Skip to content

Commit

Permalink
Merge pull request #71 from MacroEnergyProject/68-write-log-to-text-file
Browse files Browse the repository at this point in the history
Record log into file
  • Loading branch information
Betristor authored Oct 6, 2022
2 parents 929ec0f + ad15924 commit c2d0906
Show file tree
Hide file tree
Showing 83 changed files with 251 additions and 179 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ time_domain_reduction/PreCluster_Plots.jl
time_domain_reduction/TSC_Plots.jl
time_domain_reduction/TSC.jl
gurobi.log
**/output.log
Example_Systems/SmallNewEngland/ThreeZones/TDR_*
Example_Systems/RealSystemExample/ISONE_Singlezone/Manifest.toml
Example_Systems/RealSystemExample/ISONE_Singlezone/Results*
Expand Down
20 changes: 10 additions & 10 deletions Example_Systems/2020_NoEC/Run.jl
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ src_path = "../../src/"
inpath = pwd()

### Load GenX
println("Loading packages")
print_and_log("Loading packages")
push!(LOAD_PATH, src_path)

using DOLPHYN
Expand All @@ -47,30 +47,30 @@ if mysetup["TimeDomainReduction"] == 1

if mysetup["ModelH2"] == 1
if (!isfile(TDRpath*"/Load_data.csv")) || (!isfile(TDRpath*"/Generators_variability.csv")) || (!isfile(TDRpath*"/Fuels_data.csv")) || (!isfile(TDRpath*"/HSC_generators_variability.csv")) || (!isfile(TDRpath*"/HSC_load_data.csv"))
println("Clustering Time Series Data...")
print_and_log("Clustering Time Series Data...")
cluster_inputs(inpath, settings_path, mysetup)
else
println("Time Series Data Already Clustered.")
print_and_log("Time Series Data Already Clustered.")
end
else
if (!isfile(TDRpath*"/Load_data.csv")) || (!isfile(TDRpath*"/Generators_variability.csv")) || (!isfile(TDRpath*"/Fuels_data.csv"))
println("Clustering Time Series Data...")
print_and_log("Clustering Time Series Data...")
cluster_inputs(inpath, settings_path, mysetup)
else
println("Time Series Data Already Clustered.")
print_and_log("Time Series Data Already Clustered.")
end
end

end

# ### Configure solver
println("Configuring Solver")
print_and_log("Configuring Solver")
OPTIMIZER = configure_solver(mysetup["Solver"], settings_path)

# #### Running a case

# ### Load inputs
# println("Loading Inputs")
# print_and_log("Loading Inputs")
myinputs = Dict() # myinputs dictionary will store read-in data and computed parameters
myinputs = load_inputs(mysetup, inpath)

Expand All @@ -80,17 +80,17 @@ if mysetup["ModelH2"] == 1
end

# ### Generate model
# println("Generating the Optimization Model")
# print_and_log("Generating the Optimization Model")
EP = generate_model(mysetup, myinputs, OPTIMIZER)

### Solve model
println("Solving Model")
print_and_log("Solving Model")
EP, solve_time = solve_model(EP, mysetup)
myinputs["solve_time"] = solve_time # Store the model solve time in myinputs

### Write power system output

println("Writing Output")
print_and_log("Writing Output")
outpath = "$inpath/Results"
outpath=write_outputs(EP, outpath, mysetup, myinputs)

Expand Down
20 changes: 10 additions & 10 deletions Example_Systems/2030_CombEC_DETrans/Run.jl
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ src_path = "../../src/"
inpath = pwd()

### Load GenX
println("Loading packages")
print_and_log("Loading packages")
push!(LOAD_PATH, src_path)

using DOLPHYN
Expand All @@ -47,30 +47,30 @@ if mysetup["TimeDomainReduction"] == 1

if mysetup["ModelH2"] == 1
if (!isfile(TDRpath*"/Load_data.csv")) || (!isfile(TDRpath*"/Generators_variability.csv")) || (!isfile(TDRpath*"/Fuels_data.csv")) || (!isfile(TDRpath*"/HSC_generators_variability.csv")) || (!isfile(TDRpath*"/HSC_load_data.csv"))
println("Clustering Time Series Data...")
print_and_log("Clustering Time Series Data...")
cluster_inputs(inpath, settings_path, mysetup)
else
println("Time Series Data Already Clustered.")
print_and_log("Time Series Data Already Clustered.")
end
else
if (!isfile(TDRpath*"/Load_data.csv")) || (!isfile(TDRpath*"/Generators_variability.csv")) || (!isfile(TDRpath*"/Fuels_data.csv"))
println("Clustering Time Series Data...")
print_and_log("Clustering Time Series Data...")
cluster_inputs(inpath, settings_path, mysetup)
else
println("Time Series Data Already Clustered.")
print_and_log("Time Series Data Already Clustered.")
end
end

end

# ### Configure solver
println("Configuring Solver")
print_and_log("Configuring Solver")
OPTIMIZER = configure_solver(mysetup["Solver"], settings_path)

# #### Running a case

# ### Load inputs
# println("Loading Inputs")
# print_and_log("Loading Inputs")
myinputs = Dict() # myinputs dictionary will store read-in data and computed parameters
myinputs = load_inputs(mysetup, inpath)

Expand All @@ -80,17 +80,17 @@ if mysetup["ModelH2"] == 1
end

# ### Generate model
# println("Generating the Optimization Model")
# print_and_log("Generating the Optimization Model")
EP = generate_model(mysetup, myinputs, OPTIMIZER)

### Solve model
println("Solving Model")
print_and_log("Solving Model")
EP, solve_time = solve_model(EP, mysetup)
myinputs["solve_time"] = solve_time # Store the model solve time in myinputs

### Write power system output

println("Writing Output")
print_and_log("Writing Output")
outpath = "$inpath/Results"
outpath=write_outputs(EP, outpath, mysetup, myinputs)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -641,7 +641,7 @@ uuid = "d3d80556-e9d4-5f37-9878-2ab0fcc64255"
version = "7.1.1"

[[LinearAlgebra]]
deps = ["Libdl"]
deps = ["Libdl", "libblastrampoline_jll"]
uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"

[[LogExpFunctions]]
Expand All @@ -653,6 +653,12 @@ version = "0.3.0"
[[Logging]]
uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"

[[LoggingExtras]]
deps = ["Dates", "Logging"]
git-tree-sha1 = "5d4d2d9904227b8bd66386c1138cf4d5ffa826bf"
uuid = "e6f89c97-d47a-5376-807f-9c37f3926c36"
version = "0.4.9"

[[LoopVectorization]]
deps = ["ArrayInterface", "DocStringExtensions", "IfElse", "LinearAlgebra", "OffsetArrays", "Polyester", "Requires", "SLEEFPirates", "Static", "StrideArraysCore", "ThreadingUtilities", "UnPack", "VectorizationBase"]
git-tree-sha1 = "9f23789217866ad9ecd053857ef202de5edcac4b"
Expand Down Expand Up @@ -793,6 +799,10 @@ git-tree-sha1 = "ba4a8f683303c9082e84afba96f25af3c7fb2436"
uuid = "656ef2d0-ae68-5445-9ca0-591084a874a2"
version = "0.3.12+1"

[[OpenBLAS_jll]]
deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"]
uuid = "4536629a-c528-5b80-bd46-f80d51c5b363"

[[OpenLibm_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "05823500-19ac-5b8b-9628-191a04bc5112"
Expand Down Expand Up @@ -911,7 +921,7 @@ deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"]
uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"

[[Random]]
deps = ["Serialization"]
deps = ["SHA", "Serialization"]
uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"

[[Random123]]
Expand Down Expand Up @@ -1262,6 +1272,10 @@ git-tree-sha1 = "f8b75e896a326a162a4f6e998990521d8302c810"
uuid = "508c9074-7a14-5c94-9582-3d4bc1871065"
version = "0.77.0+1"

[[libblastrampoline_jll]]
deps = ["Artifacts", "Libdl", "OpenBLAS_jll"]
uuid = "8e850b90-86db-534c-a0d3-1478176c7d93"

[[libsass_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "663428b7ebaf60c60ee147f0f9466430e9959ad6"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ Gurobi = "2e9cd046-0924-5485-92f1-d5272153d98b"
Ipopt = "b6b21f68-93f8-5de0-b562-5493be1d77c9"
JuMP = "4076af6c-e467-56ae-b986-b466b2749572"
LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
Logging = "56ddb016-857b-54e1-b83d-db4d58db5568"
LoggingExtras = "e6f89c97-d47a-5376-807f-9c37f3926c36"
MathOptInterface = "b8f27783-ece8-5eb3-8dc8-9495eed66fee"
MathProgBase = "fdba3010-5040-5b88-9595-932c9decdf73"
OrdinaryDiffEq = "1dea7af3-3e70-54e6-95c3-0bf5283fa5ed"
Expand Down
55 changes: 33 additions & 22 deletions Example_Systems/SmallNewEngland/ThreeZones/Run.jl
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,34 @@ in LICENSE.txt. Users uncompressing this from an archive may not have
received this license file. If not, see <http://www.gnu.org/licenses/>.
"""

# Walk into current directory
cd(dirname(@__FILE__))


# Loading settings
using YAML

settings_path = joinpath(pwd(), "Settings")

genx_settings = joinpath(settings_path, "genx_settings.yml") #Settings YAML file path for GenX
hsc_settings = joinpath(settings_path, "hsc_settings.yml") #Settings YAML file path for HSC modelgrated model
mysetup_genx = YAML.load(open(genx_settings)) # mysetup dictionary stores GenX-specific parameters
mysetup_hsc = YAML.load(open(hsc_settings)) # mysetup dictionary stores H2 supply chain-specific parameters
global_settings = joinpath(settings_path, "global_model_settings.yml") # Global settings for inte
mysetup_global = YAML.load(open(global_settings)) # mysetup dictionary stores global settings
mysetup = Dict()
mysetup = merge(mysetup_hsc, mysetup_genx, mysetup_global) #Merge dictionary - value of common keys will be overwritten by value in global_model_settings

# Start logging
using LoggingExtras

global Log = mysetup["Log"]

if Log
logger = FileLogger(mysetup["LogFile"])
global_logger(logger)
end

# Activate environment
environment_path = "../../../package_activate.jl"
include(environment_path) #Run this line to activate the Julia virtual environment for GenX; skip it, if the appropriate package versions are installed

Expand All @@ -28,52 +50,41 @@ src_path = "../../../src/"

inpath = pwd()

### Load GenX
### Load DOLPHYN
println("Loading packages")
push!(LOAD_PATH, src_path)

using DOLPHYN
using YAML

genx_settings = joinpath(settings_path, "genx_settings.yml") #Settings YAML file path for GenX
hsc_settings = joinpath(settings_path, "hsc_settings.yml") #Settings YAML file path for HSC modelgrated model
mysetup_genx = YAML.load(open(genx_settings)) # mysetup dictionary stores GenX-specific parameters
mysetup_hsc = YAML.load(open(hsc_settings)) # mysetup dictionary stores H2 supply chain-specific parameters
global_settings = joinpath(settings_path, "global_model_settings.yml") # Global settings for inte
mysetup_global = YAML.load(open(global_settings)) # mysetup dictionary stores global settings
mysetup = Dict()
mysetup = merge( mysetup_hsc, mysetup_genx, mysetup_global) #Merge dictionary - value of common keys will be overwritten by value in global_model_settings

## Cluster time series inputs if necessary and if specified by the user
TDRpath = joinpath(inpath, mysetup["TimeDomainReductionFolder"])
if mysetup["TimeDomainReduction"] == 1

if mysetup["ModelH2"] == 1
if (!isfile(TDRpath*"/Load_data.csv")) || (!isfile(TDRpath*"/Generators_variability.csv")) || (!isfile(TDRpath*"/Fuels_data.csv")) || (!isfile(TDRpath*"/HSC_generators_variability.csv")) || (!isfile(TDRpath*"/HSC_load_data.csv"))
println("Clustering Time Series Data...")
print_and_log("Clustering Time Series Data...")
cluster_inputs(inpath, settings_path, mysetup)
else
println("Time Series Data Already Clustered.")
print_and_log("Time Series Data Already Clustered.")
end
else
if (!isfile(TDRpath*"/Load_data.csv")) || (!isfile(TDRpath*"/Generators_variability.csv")) || (!isfile(TDRpath*"/Fuels_data.csv"))
println("Clustering Time Series Data...")
print_and_log("Clustering Time Series Data...")
cluster_inputs(inpath, settings_path, mysetup)
else
println("Time Series Data Already Clustered.")
print_and_log("Time Series Data Already Clustered.")
end
end

end

# ### Configure solver
println("Configuring Solver")
print_and_log("Configuring Solver")
OPTIMIZER = configure_solver(mysetup["Solver"], settings_path)

# #### Running a case

# ### Load power system inputs
# println("Loading Inputs")
# print_and_log("Loading Inputs")
myinputs = Dict() # myinputs dictionary will store read-in data and computed parameters
myinputs = load_inputs(mysetup, inpath)

Expand All @@ -83,17 +94,17 @@ if mysetup["ModelH2"] == 1
end

# ### Generate model
# println("Generating the Optimization Model")
# print_and_log("Generating the Optimization Model")
EP = generate_model(mysetup, myinputs, OPTIMIZER)

### Solve model
println("Solving Model")
print_and_log("Solving Model")
EP, solve_time = solve_model(EP, mysetup)
myinputs["solve_time"] = solve_time # Store the model solve time in myinputs

### Write power system output

println("Writing Output")
print_and_log("Writing Output")
outpath = "$inpath/Results"
outpath=write_outputs(EP, outpath, mysetup, myinputs)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,3 +7,8 @@ WriteShadowPrices: 1 # Write shadow prices of LP or relaxed MILP; 0 = not active
OperationWrapping: 1 # Sets temporal resolution of the model; 0 = single period to represent the full year, with first-last time step linked; 1 = multiple representative periods
TimeDomainReductionFolder: "TDR_Results" # Directory name where results from time domain reduction will be saved. If results already exist here, these will be used without running time domain reduction script again.
TimeDomainReduction: 1 # Time domain reduce (i.e. cluster) inputs based on Load_data.csv, Generators_variability.csv, and Fuels_data.csv; 0 = not active (use input data as provided); 1 = active (cluster input data, or use data that has already been clustered)


# Log file
Log: true # Flag whether log directed to file, turing this on will record basic output into log file and keep console io stream
LogFile: output.log # Filename for log file
Original file line number Diff line number Diff line change
Expand Up @@ -14,3 +14,7 @@ Crossover: -1 # Barrier crossver strategy.
BarObjRng: 1e+75 # Sets the maximum absolute value of the objective function
PreDual: 0 # Decides whether presolve should pass the primal or dual linear programming problem to the LP optimization algorithm.
AggFill: 10 # Allowed fill during presolve aggregation.

# Gurobi log file
GurobiLog: true # Flag whether record gurobi log into file
GurobiLogFile: "gurobi.log" # Fulename for gurobi log
2 changes: 2 additions & 0 deletions Project.toml
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ Gurobi = "2e9cd046-0924-5485-92f1-d5272153d98b"
Ipopt = "b6b21f68-93f8-5de0-b562-5493be1d77c9"
JuMP = "4076af6c-e467-56ae-b986-b466b2749572"
LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
Logging = "56ddb016-857b-54e1-b83d-db4d58db5568"
LoggingExtras = "e6f89c97-d47a-5376-807f-9c37f3926c36"
MathOptInterface = "b8f27783-ece8-5eb3-8dc8-9495eed66fee"
MathProgBase = "fdba3010-5040-5b88-9595-932c9decdf73"
OrdinaryDiffEq = "1dea7af3-3e70-54e6-95c3-0bf5283fa5ed"
Expand Down
4 changes: 4 additions & 0 deletions julenv.jl
Original file line number Diff line number Diff line change
Expand Up @@ -52,3 +52,7 @@ Pkg.add(Pkg.PackageSpec(name="StatsBase", version="0.33.8"))
Pkg.add(Pkg.PackageSpec(name="YAML", version="0.4.7"))
Pkg.add(Pkg.PackageSpec(name="LinearAlgebra"))
Pkg.add(Pkg.PackageSpec(name="Revise"))

# Logging
Pkg.add(Pkg.PackageSpec(name="Logging"))
Pkg.add(Pkg.PackageSpec(name="LoggingExtras"))
7 changes: 7 additions & 0 deletions src/DOLPHYN.jl
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ received this license file. If not, see <http://www.gnu.org/licenses/>.
module DOLPHYN

#export package_activate
export print_and_log
export configure_settings
export configure_solver
export load_inputs
Expand Down Expand Up @@ -60,6 +61,12 @@ ModelScalingFactor = 1e+3
# LHV is used when defining a system-wide CO2 constraint for the joint hydrogen and electricity infrastructures (SystemCO2Constraint =2)
H2_LHV = 33.33 # MWh per tonne

# Logging flag
Log = true

# Auxiliary logging function
include("print_and_log.jl")

# Configure settings
include("configure_settings/configure_settings.jl")

Expand Down
Loading

0 comments on commit c2d0906

Please sign in to comment.