Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Initial python skeleton and a little workflow-related code #1

Merged
merged 1 commit into from
Dec 22, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions dispatches/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
"""
Workflow data management functionality.
"""
# promote some things from the workflow.py module
from .workflow import ManagedWorkflow
2 changes: 2 additions & 0 deletions dispatches/workflow/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
# expose selected objects in package namespace
from .workflow import ManagedWorkflow
13 changes: 13 additions & 0 deletions dispatches/workflow/rts_gmlc.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
"""
Wrappers for Prescient RTS-GMLC functions
"""
from pathlib import Path
import prescient.downloaders.rts_gmlc as rts_downloader


def download() -> Path:
"""Wraps RTS GMLC downloader.
"""
rts_downloader.download()
rts_gmlc_dir = Path(rts_downloader.rts_download_path) / "RTS-GMLC"
return rts_gmlc_dir
Empty file.
26 changes: 26 additions & 0 deletions dispatches/workflow/tests/test_workflow.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
"""
Tests for dispatches.workflow.workflow
"""
import pytest

from dispatches.workflow import ManagedWorkflow
from dispatches.workflow.workflow import Dataset, DatasetFactory


def test_managed_workflow():
wf = ManagedWorkflow("hello", "world")
assert wf.name == "hello"
assert wf.workspace_name == "world"


def test_dataset():
ds = Dataset("hello")
assert ds.name == "hello"
assert ds.meta == {}


def test_dataset_factory():
df = DatasetFactory("null")
assert df.create(hello="ignored") is None
# with unknown type, raises KeyError
pytest.raises(KeyError, DatasetFactory, "?")
88 changes: 88 additions & 0 deletions dispatches/workflow/workflow.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
"""
Managed data workflows for Prescient
"""
# stdlib
import os
# package
from . import rts_gmlc


class ManagedWorkflow:
def __init__(self, name, workspace_name):
self._name = name
self._workspace_name = workspace_name
self._datasets = {}
# TODO: create instance of DMF

@property
def name(self):
return self._name

@property
def workspace_name(self):
return self._workspace_name

def get_dataset(self, type_, **kwargs):
"""Creates and returns a dataset of the specified type. If called more than once with the
same type of dataset, then returns the previous value.
"""
ds = self._datasets.get(type_, None)
if ds is not None:
return ds
dsf = DatasetFactory(type_, workflow=self)
ds = dsf.create(**kwargs)
self._datasets[type_] = ds
# TODO: register new dataset with DMF
return ds


class Dataset:
def __init__(self, name):
self.name = name
self._meta = {}

@property
def meta(self):
return self._meta.copy()

def add_meta(self, key, value):
self._meta[key] = value

def __str__(self):
lines = [
"Metadata",
"--------"
]
for key, value in self._meta.items():
lines.append("%s:" % key)
lines.append(str(value))
return "\n".join(lines)


class DatasetFactory:
def __init__(self, type_, workflow=None):
self._wf = workflow
try:
self.create = self._get_factory_function(type_)
except KeyError:
raise KeyError("Cannot create dataset of type '%s'" % type_)

@classmethod
def _get_factory_function(cls, name):
# This could be more dynamic..
if name == "rts-gmlc":

def download_fn(**kwargs):
rts_gmlc_dir = rts_gmlc.download()
dataset = Dataset(name)
dataset.add_meta("directory", rts_gmlc_dir)
dataset.add_meta("files", os.listdir(rts_gmlc_dir))
return dataset

return download_fn
elif name == "null":
def fn(**kwargs):
return None # XXX: or do we need a NullDataset subclass?
return fn
else:
raise KeyError(name)
118 changes: 118 additions & 0 deletions examples/dmf_prescient_tutorial.ipynb
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {
"collapsed": true,
"pycharm": {
"name": "#%% md\n"
}
},
"source": [
"Re-imagining the Prescient tutorial using a data management framework\n",
"\n",
"\n"
]
},
{
"cell_type": "markdown",
"source": [
"Downloading the RTS-GMLC data"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%% md\n"
}
}
},
{
"cell_type": "code",
"execution_count": 2,
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"RTS-GMLC already downloaded to C:\\Users\\dkgun\\src\\dispatches\\dangunter\\Prescient\\downloads\\rts_gmlc\\RTS-GMLC. If you would like re-download it, delete the directory C:\\Users\\dkgun\\src\\dispatches\\dangunter\\Prescient\\downloads\\rts_gmlc\\RTS-GMLC.\n",
"Files:\n",
".git\n",
".gitignore\n",
".gitmodules\n",
"CONTRIBUTING.md\n",
"node_re_basemap.png\n",
"README.md\n",
"RTS-GMLC.pdf\n",
"RTS-GMLC_updates.md\n",
"RTS_Data\n",
"rts_layers.png\n"
]
}
],
"source": [
"\n",
"import prescient.downloaders.rts_gmlc as rts_downloader\n",
"\n",
"# NEW\n",
"from prescient.workflow import ManagedWorkflow\n",
"from dispatches.workflow.rts_gmlc import DatasetFactory\n",
"\n",
"# Create a new managed workflow in a DMF workspace with the given name\n",
"workflow = ManagedWorkflow(\"prescient_tutorial_workflow\", \"prescient_tutorial_workspace\")\n",
"\n",
"\n",
"# Return the metadata of the dataset as a dictionary-like object\n",
"ds = workflow.get_dataset(\"rts-gmlc\")\n",
"# print(\"dataset:\\n%s\" % ds)\n",
"# Print out the file-list, as in the original tutorial\n",
"print(\"Files:\")\n",
"try:\n",
" files = ds.meta[\"files\"]\n",
"except KeyError:\n",
" print(\"No files\")\n",
"else:\n",
" for filename in files:\n",
" print(filename)"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
}
}
},
{
"cell_type": "code",
"execution_count": 2,
"outputs": [],
"source": [],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
}
}
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.6"
}
},
"nbformat": 4,
"nbformat_minor": 0
}
78 changes: 78 additions & 0 deletions setup.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
"""
Project setup with setuptools
"""

# Always prefer setuptools over distutils
from setuptools import setup, find_namespace_packages
import pathlib

cwd = pathlib.Path(__file__).parent.resolve() # this will come in handy, probably

long_description = """ProteusLib is an open-source, integrated suite of predictive multi-scale models
for design and optimization of water treatment processes and systems. Specifically, ProteusLib is a new
library of water treatment-specific property, process unit, and network models that depend on the IDAES Platform,
an open source, next generation process systems engineering platform developed at the National Energy Technology
Laboratory with other partners. The ProteusLib project is funded by the NAWI as a part of U.S. Department of
Energy’s Energy-Water Desalination Hub. The goal of ProteusLib is to assist the hub and the broader water R&D
community in assessing existing and emerging water treatment technologies by 1) providing predictive capabilities
involving the design, optimization, and performance of water treatment systems that will lead to improved energy
efficiency and lower cost, 2) advancing the state of the art for the design of water treatment components, systems
and networks to be comparable with, or even surpass, that in the chemical industry, and 3) disseminating these tools
for active use by water treatment researchers and engineers.""".replace("\n", " ").strip()

# Arguments marked as "Required" below must be included for upload to PyPI.
# Fields marked as "Optional" may be commented out.

setup(
name='dispatches',
url='https://github.com/gmlc-dispatches/dispatches',
version='0.0.1',
description='GMLC DISPATCHES software tools',
long_description=long_description,
long_description_content_type='text/plain',
author='DISPATCHES team',
# Classifiers help users find your project by categorizing it.
#
# For a list of valid classifiers, see https://pypi.org/classifiers/
classifiers=[
# 3 - Alpha
# 4 - Beta
# 5 - Production/Stable
'Development Status :: 3 - Alpha',
"Intended Audience :: End Users/Desktop",
"Intended Audience :: Science/Research",
"License :: OSI Approved :: BSD License",
"Natural Language :: English",
"Operating System :: MacOS",
"Operating System :: Microsoft :: Windows",
"Operating System :: Unix",
"Programming Language :: Python",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.6",
"Programming Language :: Python :: 3.7",
"Programming Language :: Python :: 3.8",
"Programming Language :: Python :: Implementation :: CPython",
"Topic :: Scientific/Engineering :: Mathematics",
"Topic :: Scientific/Engineering :: Chemistry",
"Topic :: Software Development :: Libraries :: Python Modules",
'Programming Language :: Python :: 3 :: Only',
],
keywords="market simulation, chemical engineering, process modeling, hybrid power systems",
packages=find_namespace_packages(),
python_requires='>=3.6, <4',
install_requires=[
'pytest', # technically developer, but everyone likes tests
'prescient @ git+https://github.com/grid-parity-exchange/Prescient.git'
],
dependency_links=['git+https://github.com/grid-parity-exchange/Prescient.git#egg=prescient'],
extras_require={
'dev': [
'pytest-cov'
],
},
package_data={ # Optional
"": [
"*.json",
],
},
)