Merge pull request #53 from e4st-dev/feat-12-iter

Merge the iteration framework and a few tests into main.

Project.toml

@@ -17,12 +17,15 @@ Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
 YAML = "ddb6d928-2868-570f-bddf-ab3f9cf99eb6"
 
+
 [compat]
 julia = "1.0"
 
 [extras]
 HiGHS = "87dc4568-4c63-4d18-b0c0-bb2238e4078b"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+BasicInterpolators = "26cce99e-4866-4b6d-ab74-862489e035e0"
+
 
 [targets]
-test = ["Test", "HiGHS"]
+test = ["Test", "HiGHS", "BasicInterpolators"]

src/E4ST.jl

@@ -33,6 +33,7 @@ include("types/Modification.jl")
 include("types/Policy.jl")
 include("io/config.jl")
 include("io/data.jl")
+include("io/util.jl")
 include("io/demand.jl")
 include("io/results.jl")
 include("model/setup.jl")
@@ -54,40 +55,42 @@ function run_e4st(config)
 
     start_logging!(config)
     log_info(config)
-    @info "Config saved to: $(config[:out])"
+    @info "Config saved to: $(config[:out_path])"
 
-    data = load_data(config)
+    data  = load_data(config)
+    model = setup_model(config, data)
 
-    iter = true
 
-    while iter
-        # Setup the model and save the results
+    optimize!(model)
+    check(model)
+
+    all_results = []
+
+    parse_results!(config, data, model, all_results)  
+    process!(config, all_results)
+
+    iter = get_iterator(config)
+
+    while should_iterate(iter, config, data, model, all_results)
+        iterate!(iter, config, data, model, all_results)
+        data = should_reload_data(iter) ? load_data(config) : data
+
         model = setup_model(config, data)
 
         # Optimize and save
         optimize!(model)
 
         check(model)
-        results = parse_results(config, data, model)  
-        process!(config, results)
-
-        iter = should_iterate(config, data, model)
-        iter && iterate!(config, data, model)
+        parse_results!(config, data, model, all_results)
+        process!(config, all_results)
     end
-    return results
+
+    stop_logging!(config)
+    return all_results
 end
 
 run_e4st(path::String) = run_e4st(load_config(path))
 
-"""
-    reload_policies!() -> nothing
-
-Reloads the any `Policy` types so that `PolicyFromString` will work.
-"""
-function reload_policies!()
-    reload_types!(Policy)
-end
-
 global STR2TYPE = Dict{String, Type}()
 global SYM2TYPE = Dict{Symbol, Type}()
 global STR2OPT= Dict{String, Type}()
@@ -121,6 +124,7 @@ Loads all types associated with E4ST so that the type will accessible by string
 """
 function reload_types!()
     reload_types!(Modification)
+    reload_types!(Iterable)
 end
 function reload_types!(::Type{T}) where T
     global STR2TYPE

src/io/config.jl

@@ -27,6 +27,7 @@ The Config File is a file that fully specifies all the necessary information.  N
 
 ## Optional Fields:
 * `af_file` - The filepath (relative or absolute) to the availability factor table.  See [`load_af_table!`](@ref)
+* `iter` - The [`Iterable`](@ref) object to specify the way the sim should iterate.  If nothing specified, defaults to run a single time.  Specify the `Iterable` type, and all keyword arguments.
 * `demand_shape_file` - a file for specifying the hourly shape of demand elements.  See [`load_demand_shape_table!`](@ref)
 * `demand_match_file` - a file for specifying annual demanded energy to match for sets  See [`load_demand_match_table!`](@ref)
 * `demand_add_file` - a file for specifying additional demanded energy, after matching.  See [`load_demand_add_table!`](@ref)
@@ -35,7 +36,6 @@ The Config File is a file that fully specifies all the necessary information.  N
 * `save_model_presolve` - A boolean specifying whether or not to save the model before solving it, for later use (i.e. by specifying a `model_presolve_file` for future sims). Defaults to `true`
 * `model_presolve_file` - The filepath (relative or absolute) to the unsolved model.  If this is provided, it will use this instead of creating a new model.
 
-
 ## Example Config File
 ```yaml
 $(read_sample_config_file())
@@ -51,7 +51,8 @@ function load_config(filename)
     check_required_fields!(config)
     make_paths_absolute!(config, filename)
     make_out_path!(config)
-    convert_types!(config, :mods)
+    convert_mods!(config)
+    convert_iter!(config)
     return config
 end
 
@@ -227,6 +228,10 @@ function getmods(config)
     config[:mods]
 end
 
+function get_iterator(config)
+    return get(config, :iter, RunOnce())
+end
+
 # Helper Functions
 ################################################################################
 function required_fields()
@@ -295,11 +300,17 @@ function make_out_path!(config)
     mkpath(config[:out_path])
 end
 
-function convert_types!(config, sym::Symbol)
-    if isnothing(config[sym])
-        config[sym] = OrderedDict{Symbol, Modification}()
+function convert_mods!(config)
+    if ~haskey(config, :mods) || isnothing(config[:mods])
+        config[:mods] = OrderedDict{Symbol, Modification}()
         return
     end
-    config[sym] = OrderedDict(key=>Modification(key=>val) for (key,val) in config[sym])
+    config[:mods] = OrderedDict(key=>Modification(key=>val) for (key,val) in config[:mods])
     return
 end
+
+function convert_iter!(config)
+    haskey(config, :iter) || return
+    config[:iter] = Iterable(config[:iter])
+    return
+end

src/io/data.jl

@@ -122,6 +122,9 @@ export load_gen_table!
 Sets up the generator table.
 """
 function setup_gen_table!(config, data)
+    bus = get_bus_table(data)
+    gen = get_gen_table(data)
+    leftjoin!(gen, bus, on=:bus_idx)
 end
 export setup_gen_table!
 
@@ -592,29 +595,50 @@ export summarize_af_table
 """
     get_gen_table(data)
 
-Returns gen data table
+Returns gen table
+
+    get_gen_table(data, pairs...)
+
+Returns a SubDataFrame of the gen table where each pair represents a filter condition.  See [`filter_view`](@ref) for reference.
 """
 function get_gen_table(data) 
     return data[:gen]::DataFrame
 end
+function get_gen_table(data, args...)
+    return filter_view(get_gen_table(data), args...)
+end
 
 """
     get_branch_table(data)
 
 Returns table of the transmission lines (branches) from data. 
+
+    get_branch_table(data, pairs...)
+
+Returns a SubDataFrame of the branch table where each pair represents a filter condition.  See [`filter_view`](@ref) for reference.
 """
 function get_branch_table(data) 
     return data[:branch]::DataFrame
 end
+function get_branch_table(data, args...)
+    return filter_view(get_branch_table(data), args...)
+end
 
 """
     get_bus_table(data)
 
 Returns the bus data table
+
+    get_bus_table(data, pairs...)
+
+Returns a SubDataFrame of the bus table where each pair represents a filter condition.  See [`filter_view`](@ref) for reference.
 """
 function get_bus_table(data)
     data[:bus]::DataFrame
 end
+function get_bus_table(data, args...)
+    return filter_view(get_bus_table(data), args...)
+end
 
 """
     get_hours_table(data)
@@ -635,7 +659,7 @@ function get_demand_table(data)
     return data[:demand_table]::DataFrame
 end
 function get_demand_table(data, args...)
-    return filter_view_table(get_demand_table(data), args...)
+    return filter_view(get_demand_table(data), args...)
 end
 export get_demand_table
 
@@ -867,7 +891,7 @@ end
 Returns the number of hours in a year spent at the `hour_idx` representative hour
 """
 function get_hour_weight(data, hour_idx::Int64)
-    return get_hour_weights(data)[hour_idx, :hours]
+    return get_hour_weights(data)[hour_idx]
 end
 export get_num_hours, get_hour_weights, get_hour_weight
 
@@ -896,27 +920,27 @@ export get_num_years, get_years
 
     filter_view(table::DataFrame, pairs...) -> v::SubDataFrame
 
-Return a `SubDataFrame` containing each row of `table` such that for each `(field,value)` pair in `pairs`, `row.field==value`.
+Return a `SubDataFrame` containing each row of `table` such that for each `(field,value)` pair in `pairs`, `row[field]==value`.
 """
-function filter_view_table(table::DataFrame, pairs::Pair...)
+function filter_view(table::DataFrame, pairs::Pair...)
     v = view(table,:,:)
     for (field, value) in pairs
         field isa AbstractString && isempty(field) && continue
         isempty(value) && continue
         v = filter(field=>==(value), v, view=true)
         isempty(v) && break
     end
-    return v
+    return v::SubDataFrame
 end
-function filter_view_table(table::DataFrame, pairs)
+function filter_view(table::DataFrame, pairs)
     v = view(table,:,:)
     for (field, value) in pairs
         field isa AbstractString && isempty(field) && continue
         isempty(value) && continue
         v = filter(field=>==(value), v, view=true)
         isempty(v) && break
     end
-    return v
+    return v::SubDataFrame
 end
 
 # Containers

src/io/util.jl

@@ -0,0 +1,124 @@
+
+"""
+    get_year_idxs(data, year_idxs) -> idxs
+
+Converts `year_idxs` into a usable set of indices that can index into `get_years(data)`.  `year_idxs` can be any of the following types:
+* `Colon`
+* `Int64`
+* `AbstractVector{Int64}`
+* `AbstractString` - Representing the year, i.e. "y2020"
+* `AbstractVector{<:AbstractString}` - a vector of strings representing the year, i.e. "y2020"
+"""
+function get_year_idxs(data, year_idxs::Colon)
+    year_idxs
+end
+function get_year_idxs(data, year_idxs::AbstractVector{Int64})
+    year_idxs
+end
+function get_year_idxs(data, year_idxs::Int64)
+    year_idxs
+end
+function get_year_idxs(data, year_idxs::AbstractString)
+    return findfirst(==(year_idxs), get_years(data))
+end
+function get_year_idxs(data, year_idxs::AbstractVector{<:AbstractString})
+    yrs = get_years(data)
+    return map(y->findfirst(==(y), yrs), year_idxs)
+end
+export get_year_idxs
+
+
+"""
+    get_hour_idxs(data, hour_idxs)
+
+Converts `hour_idxs` into a usable set of indices that can index into hourly data.  `hour_idxs` can be any of the following types:
+* `Colon`
+* `Int64`
+* `AbstractVector{Int64}`    
+"""
+function get_hour_idxs(data, hour_idxs::Colon)
+    hour_idxs
+end
+function get_hour_idxs(data, hour_idxs::AbstractVector{Int64})
+    hour_idxs
+end
+function get_hour_idxs(data, hour_idxs::Int64)
+    hour_idxs
+end
+export get_hour_idxs
+
+"""
+    table_rows(table, idxs) -> row_idxs
+
+Returns row indices of the passed-in table that correspond to idxs, where `idxs` can be:
+* `::Colon` - all rows
+* `::Int64` - a single row
+* `::AbstractVector{Int64}` - a list of rows
+* `p::Pair` - returns a Vector containing the index of each row for which row[p[1]] == p[2]
+* `pairs`, an iterator of `Pair`s - returns a Vector containing the indices which satisfy all the pairs as above.
+
+See also [`filter_view`](@ref)
+"""
+function table_rows(table, idxs::Colon)
+    return idxs
+end
+
+function table_rows(table, idxs::AbstractVector{Int64})
+    return idxs
+end
+
+function table_rows(table, idxs::Int64)
+    return idxs
+end
+
+function table_rows(table, pairs)
+    row_idxs = Int64[i for i in 1:nrow(table)]
+    for pair in pairs
+        key, val = pair
+        v = table[!,key]
+        comp = comparison(val, v)
+        filter!(row_idx->comp(v[row_idx]), row_idxs)
+    end
+
+    return row_idxs
+end
+function table_rows(table, pair::Pair)
+    row_idxs = Int64[i for i in 1:nrow(table)]
+    key, val = pair
+    v = table[!, key]
+    comp = comparison(val, v)
+    filter!(row_idx->comp(v[row_idx]), row_idxs)
+    return row_idxs
+end
+
+
+"""
+    comparison(value, v) -> comp
+
+Returns the appropriate comparison function for `value` to be compared to each member of `v`.
+
+    comparison(value, ::Type)
+
+Returns the appropriate comparison function for `value` to be compared to the 2nd argument type.
+"""
+function comparison(value, v::AbstractVector)
+    comparison(value, eltype(v))
+end
+
+function comparison(value::Function, ::Type)
+    return value
+end
+
+function comparison(value::String, ::Type{<:Integer})
+    num = parse(Int, value)
+    return ==(num)
+end
+
+function comparison(value::Tuple{<:Real, <:Real}, ::Type{<:Real})
+    lo, hi = value
+    return x -> lo <= x <= hi
+end
+
+function comparison(value, ::Type)
+    return ==(value)
+end