chore: merge develop into master for v0.2.1 release

.dockerignore

@@ -1,3 +1,9 @@
 .git
+.github
 .pytest_cache
+**/__pycache__
 .tox
+.env
+.venv
+Dockerfile
+.dockerignore

.github/workflows/docker-build.yml

@@ -0,0 +1,26 @@
+name: Publish docker image
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - 'develop'
+
+jobs:
+  push_to_registry:
+    name: Push Docker image to GitHub Packages
+    runs-on: ubuntu-latest
+    steps:
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.CR_PAT }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          push: true
+          tags: |
+            ghcr.io/breakthrough-energy/reisejl:latest

.github/workflows/test.yml

@@ -1,24 +1,49 @@
-name: Pytest
+name: Test
 
 on: push
 
 jobs:
-  test:
+  test-julia:
+    name: Julia ${{ matrix.version }}
+    if: "!contains(github.event.head_commit.message, 'skip_ci')"
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        version:
+          - '1.5'
+          - 'nightly'
+        os:
+          - ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: julia-actions/setup-julia@v1
+        with:
+          version: ${{ matrix.version }}
+      - uses: actions/cache@v1
+        with:
+          path: ~/.julia/artifacts
+          key: ${{ runner.os }}-artifacts-${{ hashFiles('**/Project.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-artifacts-
+      - uses: julia-actions/julia-buildpkg@v1
+      - uses: julia-actions/julia-runtest@v1
+
+  test-python:
+    name: Python ${{ matrix.version }}
     if: "!contains(github.event.head_commit.message, 'skip_ci')"
     runs-on: ubuntu-latest
     strategy:
         fail-fast: false
         matrix:
-          python-version: [3.7, 3.8]
-
-    name: Python ${{ matrix.python-version }}
+          version: [3.7, 3.8]
     steps:
       - uses: actions/checkout@v2
 
-      - name: Set up Python ${{ matrix.python-version }}
+      - name: Set up Python ${{ matrix.version }}
         uses: actions/setup-python@v2
         with:
-          python-version: ${{ matrix.python-version }}
+          python-version: ${{ matrix.version }}
 
       - run: python -m pip install --upgrade pip tox
       - run: tox -e pytest

Dockerfile

@@ -19,10 +19,25 @@ ENV PATH="$PATH:/usr/share/julia-1.5.3/bin" \
     FLASK_APP=pyreisejl/utility/app.py
 
 WORKDIR /app
-COPY . .
+COPY requirements.txt .
+RUN pip install -U pip; pip install -r requirements.txt
 
-RUN julia -e 'using Pkg; Pkg.activate("."); Pkg.instantiate(); Pkg.add("Gurobi"); import Gurobi; using REISE' && \
-    pip install -r requirements.txt
+COPY Project.toml .
+COPY Manifest.toml .
+RUN mkdir src && touch src/REISE.jl
 
+RUN julia -e 'using Pkg; \
+	Pkg.activate("."); \
+	Pkg.instantiate(); \
+	Pkg.add("Gurobi"); \
+	Pkg.add("GLPK")'
+
+COPY src src
+
+RUN julia -e 'import Gurobi; \
+	import GLPK; \
+	using REISE'
+
+COPY pyreisejl pyreisejl
 
 CMD ["flask", "run", "--host", "0.0.0.0"]

pyreisejl/utility/app.py

@@ -1,3 +1,5 @@
+import os
+import sys
 from pathlib import Path
 from subprocess import PIPE, Popen
 
@@ -12,7 +14,7 @@
 Example request:
 
 curl -XPOST http://localhost:5000/launch/1234
-curl -XPOST http://localhost:5000/launch/1234?threads=42
+curl -XPOST http://localhost:5000/launch/1234?threads=4&solver=glpk
 curl http://localhost:5000/status/1234
 """
 
@@ -26,23 +28,44 @@ def get_script_path():
     return str(path_to_script)
 
 
-@app.route("/launch/<int:scenario_id>", methods=["POST"])
-def launch_simulation(scenario_id):
-    cmd_call = ["python3", "-u", get_script_path(), str(scenario_id), "--extract-data"]
-    threads = request.args.get("threads", None)
+def launch_simulation(scenario_id, threads=None, solver=None):
+    cmd = [
+        sys.executable,
+        "-u",
+        get_script_path(),
+        str(scenario_id),
+        "--extract-data",
+    ]
 
     if threads is not None:
-        cmd_call.extend(["--threads", str(threads)])
+        cmd.extend(["--threads", str(threads)])
+
+    if solver is not None:
+        cmd.extend(["--solver", solver])
 
-    proc = Popen(cmd_call, stdout=PIPE, stderr=PIPE, start_new_session=True)
+    new_env = os.environ.copy()
+    new_env["PYTHONPATH"] = str(Path(__file__).parent.parent.parent.absolute())
+    proc = Popen(cmd, stdout=PIPE, stderr=PIPE, start_new_session=True, env=new_env)
     entry = SimulationState(scenario_id, proc)
     state.add(entry)
-    return jsonify(entry.as_dict())
+    return entry.as_dict()
+
+
+def check_progress():
+    return state.as_dict()
+
+
+@app.route("/launch/<int:scenario_id>", methods=["POST"])
+def handle_launch(scenario_id):
+    threads = request.args.get("threads", None)
+    solver = request.args.get("solver", None)
+    entry = launch_simulation(scenario_id, threads, solver)
+    return jsonify(entry)
 
 
 @app.route("/list")
 def list_ongoing():
-    return jsonify(state.as_dict())
+    return jsonify(check_progress())
 
 
 @app.route("/status/<int:scenario_id>")

pyreisejl/utility/call.py

@@ -1,21 +1,14 @@
 import os
-from time import time
-
-import pandas as pd
 
 from pyreisejl.utility import const, parser
 from pyreisejl.utility.extract_data import extract_scenario
 from pyreisejl.utility.helpers import (
-    InvalidDateArgument,
-    InvalidInterval,
     WrongNumberOfArguments,
-    extract_date_limits,
     get_scenario,
     insert_in_file,
     sec2hms,
-    validate_time_format,
-    validate_time_range,
 )
+from pyreisejl.utility.launchers import get_launcher
 
 
 def _record_scenario(scenario_id, runtime):
@@ -34,111 +27,6 @@ def _record_scenario(scenario_id, runtime):
     )
 
 
-class Launcher:
-    """Parent class for solver-specific scenario launchers.
-
-    :param str start_date: start date of simulation as 'YYYY-MM-DD HH:MM:SS',
-        where HH, MM, and SS are optional.
-    :param str end_date: end date of simulation as 'YYYY-MM-DD HH:MM:SS',
-        where HH, MM, and SS are optional.
-    :param int interval: length of each interval in hours
-    :param str input_dir: directory with input data
-    :raises InvalidDateArgument: if start_date is posterior to end_date
-    :raises InvalidInterval: if the interval doesn't evently divide the given date range
-    """
-
-    def __init__(self, start_date, end_date, interval, input_dir):
-        """Constructor."""
-        # extract time limits from 'demand.csv'
-        with open(os.path.join(input_dir, "demand.csv")) as profile:
-            min_ts, max_ts, freq = extract_date_limits(profile)
-
-        dates = pd.date_range(start=min_ts, end=max_ts, freq=freq)
-
-        start_ts = validate_time_format(start_date)
-        end_ts = validate_time_format(end_date, end_date=True)
-
-        # make sure the dates are within the time frame we have data for
-        validate_time_range(start_ts, min_ts, max_ts)
-        validate_time_range(end_ts, min_ts, max_ts)
-
-        if start_ts > end_ts:
-            raise InvalidDateArgument(
-                f"The start date ({start_ts}) cannot be after the end date ({end_ts})."
-            )
-
-        # Julia starts at 1
-        start_index = dates.get_loc(start_ts) + 1
-        end_index = dates.get_loc(end_ts) + 1
-
-        # Calculate number of intervals
-        ts_range = end_index - start_index + 1
-        if ts_range % interval > 0:
-            raise InvalidInterval(
-                "This interval does not evenly divide the given date range."
-            )
-        self.start_index = start_index
-        self.interval = interval
-        self.n_interval = int(ts_range / interval)
-        self.input_dir = input_dir
-        print("Validation complete!")
-
-    def _print_settings(self):
-        print("Launching scenario with parameters:")
-        print(
-            {
-                "interval": self.interval,
-                "n_interval": self.n_interval,
-                "start_index": self.start_index,
-                "input_dir": self.input_dir,
-                "execute_dir": self.execute_dir,
-                "threads": self.threads,
-            }
-        )
-
-    def launch_scenario(self):
-        # This should be defined in sub-classes
-        raise NotImplementedError
-
-
-class GurobiLauncher(Launcher):
-    def launch_scenario(self, execute_dir=None, threads=None, solver_kwargs=None):
-        """Launches the scenario.
-
-        :param None/str execute_dir: directory for execute data. None defaults to an
-            execute folder that will be created in the input directory
-        :param None/int threads: number of threads to use.
-        :param None/dict solver_kwargs: keyword arguments to pass to solver (if any).
-        :return: (*int*) runtime of scenario in seconds
-        """
-        self.execute_dir = execute_dir
-        self.threads = threads
-        self._print_settings()
-        # Import these within function because there is a lengthy compilation step
-        from julia.api import Julia
-
-        Julia(compiled_modules=False)
-        from julia import Gurobi  # noqa: F401
-        from julia import REISE
-
-        start = time()
-        REISE.run_scenario_gurobi(
-            interval=self.interval,
-            n_interval=self.n_interval,
-            start_index=self.start_index,
-            inputfolder=self.input_dir,
-            outputfolder=self.execute_dir,
-            threads=self.threads,
-        )
-        end = time()
-
-        runtime = round(end - start)
-        hours, minutes, seconds = sec2hms(runtime)
-        print(f"Run time: {hours}:{minutes:02d}:{seconds:02d}")
-
-        return runtime
-
-
 def main(args):
     # Get scenario info if using PowerSimData
     if args.scenario_id:
@@ -162,13 +50,17 @@ def main(args):
         )
         raise WrongNumberOfArguments(err_str)
 
-    launcher = GurobiLauncher(
+    launcher = get_launcher(args.solver)(
         args.start_date,
         args.end_date,
         args.interval,
         args.input_dir,
+        execute_dir=args.execute_dir,
+        threads=args.threads,
+        julia_env=args.julia_env,
+        num_segments=args.linearization_segments,
     )
-    runtime = launcher.launch_scenario(args.execute_dir, args.threads)
+    runtime = launcher.launch_scenario()
 
     # If using PowerSimData, record the runtime
     if args.scenario_id:

pyreisejl/utility/const.py

@@ -1,9 +1,13 @@
-import posixpath
+import os
+from pathlib import Path
 
-DATA_ROOT_DIR = "/mnt/bes/pcm"
+if os.getenv("DEPLOYMENT_MODE") is not None:
+    DATA_ROOT_DIR = os.path.join(Path.home(), "ScenarioData", "")
+else:
+    DATA_ROOT_DIR = "/mnt/bes/pcm"
 
-SCENARIO_LIST = posixpath.join(DATA_ROOT_DIR, "ScenarioList.csv")
-EXECUTE_LIST = posixpath.join(DATA_ROOT_DIR, "ExecuteList.csv")
-EXECUTE_DIR = posixpath.join(DATA_ROOT_DIR, "tmp")
-INPUT_DIR = posixpath.join(DATA_ROOT_DIR, "data", "input")
-OUTPUT_DIR = posixpath.join(DATA_ROOT_DIR, "data", "output")
+SCENARIO_LIST = os.path.join(DATA_ROOT_DIR, "ScenarioList.csv")
+EXECUTE_LIST = os.path.join(DATA_ROOT_DIR, "ExecuteList.csv")
+EXECUTE_DIR = os.path.join(DATA_ROOT_DIR, "tmp")
+INPUT_DIR = os.path.join(DATA_ROOT_DIR, "data", "input")
+OUTPUT_DIR = os.path.join(DATA_ROOT_DIR, "data", "output")

pyreisejl/utility/extract_data.py

@@ -113,6 +113,12 @@ def extract_data(results):
             extraction_vars |= {"load_shed"}
         except KeyError:
             pass
+        try:
+            temps["load_shift_up"] = output_mpc["flexible_demand"]["load_shift_up"].T
+            temps["load_shift_dn"] = output_mpc["flexible_demand"]["load_shift_dn"].T
+            extraction_vars |= {"load_shift_up", "load_shift_dn"}
+        except KeyError:
+            pass
 
         # Extract which number result currently being processed
         i = result_num(filename)
@@ -233,6 +239,8 @@ def _get_outputs_from_converted(matfile):
         "pf": case.mpc.branchid,
         "lmp": case.mpc.bus[:, 0].astype(np.int64),
         "load_shed": case.mpc.bus[:, 0].astype(np.int64),
+        "load_shift_up": case.mpc.bus[:, 0].astype(np.int64),
+        "load_shift_dn": case.mpc.bus[:, 0].astype(np.int64),
         "congu": case.mpc.branchid,
         "congl": case.mpc.branchid,
     }