Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Introduce DirectoryObject and FileObject in Workflow #725

Merged
merged 40 commits into from
Jun 27, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
40 commits
Select commit Hold shift + click to select a range
ab6f7af
add files.py
samwaseda Jun 19, 2023
5d2cad1
add file object
samwaseda Jun 19, 2023
ddaed6c
Format black
pyiron-runner Jun 19, 2023
1aa1e22
Merge remote-tracking branch 'origin' into directory
samwaseda Jun 20, 2023
8eae5de
resolve conflicts
samwaseda Jun 20, 2023
e8f8059
add read
samwaseda Jun 20, 2023
e98a624
add warning
samwaseda Jun 20, 2023
485acff
resolve coflicts
samwaseda Jun 20, 2023
4b7d625
add directory test
samwaseda Jun 20, 2023
8136858
add directory test
samwaseda Jun 20, 2023
cb1dff7
add test_write
samwaseda Jun 20, 2023
04410b1
categorize files
samwaseda Jun 20, 2023
6f39e59
categorize files
samwaseda Jun 20, 2023
ac0a56c
add test_path
samwaseda Jun 20, 2023
c7150e1
add test_path
samwaseda Jun 20, 2023
53e0217
add tests for is_file and delete
samwaseda Jun 20, 2023
e097963
move the location of `create_file`
samwaseda Jun 21, 2023
7c08a95
save current changes
samwaseda Jun 22, 2023
dc7e8b6
update
samwaseda Jun 22, 2023
25527a4
resolve conflict
samwaseda Jun 22, 2023
8bb2195
resolve conflicts
samwaseda Jun 22, 2023
bbd4fe7
change default mode
samwaseda Jun 22, 2023
435e787
Merge branch 'submittable_workflow' into directory
samwaseda Jun 23, 2023
ec31baa
resolve conflicts
samwaseda Jun 26, 2023
67a803a
Update tests/unit/workflow/test_workflow.py
samwaseda Jun 27, 2023
52e29d0
update tests
samwaseda Jun 27, 2023
67801b7
update tests
samwaseda Jun 27, 2023
530c85d
add test for node to raise an error for not having a parent workflow
samwaseda Jun 27, 2023
8ddb37a
add working directory tests for node
samwaseda Jun 27, 2023
57d3106
Format black
pyiron-runner Jun 27, 2023
8120324
delete working directory afterwards
samwaseda Jun 27, 2023
0955446
Merge branch 'directory' of github.com:pyiron/pyiron_contrib into dir…
samwaseda Jun 27, 2023
e9f5f23
remove unused warnings
samwaseda Jun 27, 2023
e8e5ca7
remove unused FileObject
samwaseda Jun 27, 2023
1063ee4
remove unused Server
samwaseda Jun 27, 2023
08e5444
satisfy codacy
samwaseda Jun 27, 2023
f83e335
resolve windows problem
samwaseda Jun 27, 2023
2c89272
remove one test
samwaseda Jun 27, 2023
5ce34f7
add try except
samwaseda Jun 27, 2023
67b6682
replace file paths
samwaseda Jun 27, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
99 changes: 99 additions & 0 deletions pyiron_contrib/workflow/files.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
from pathlib import Path


def delete_files_and_directories_recursively(path):
if not path.exists():
return
for item in path.rglob("*"):
if item.is_file():
item.unlink()
else:
delete_files_and_directories_recursively(item)
path.rmdir()


def categorize_folder_items(folder_path):
types = [
"dir",
"file",
"mount",
"symlink",
"block_device",
"char_device",
"fifo",
"socket",
]
results = {t: [] for t in types}

for item in folder_path.iterdir():
for tt in types:
try:
if getattr(item, f"is_{tt}")():
results[tt].append(str(item))
except NotImplementedError:
pass
return results


class DirectoryObject:
def __init__(self, directory):
self.path = Path(directory)
self.create()

def create(self):
self.path.mkdir(parents=True, exist_ok=True)

def delete(self):
delete_files_and_directories_recursively(self.path)

def list_content(self):
return categorize_folder_items(self.path)

def __len__(self):
return sum([len(cc) for cc in self.list_content().values()])

def __repr__(self):
return f"DirectoryObject(directory='{self.path}')\n{self.list_content()}"

def get_path(self, file_name):
return self.path / file_name

def file_exists(self, file_name):
return self.get_path(file_name).is_file()

def write(self, file_name, content, mode="w"):
with self.get_path(file_name).open(mode=mode) as f:
f.write(content)

def create_subdirectory(self, path):
return DirectoryObject(self.path / path)

def create_file(self, file_name):
return FileObject(file_name, self)


class FileObject:
def __init__(self, file_name: str, directory: DirectoryObject):
samwaseda marked this conversation as resolved.
Show resolved Hide resolved
self.directory = directory
self._file_name = file_name

@property
def file_name(self):
return self._file_name

@property
def path(self):
return self.directory.path / Path(self._file_name)

def write(self, content, mode="x"):
self.directory.write(file_name=self.file_name, content=content, mode=mode)

def read(self, mode="r"):
with open(self.path, mode=mode) as f:
return f.read()

def is_file(self):
return self.directory.file_exists(self.file_name)

def delete(self):
self.path.unlink()
15 changes: 15 additions & 0 deletions pyiron_contrib/workflow/node.py
Original file line number Diff line number Diff line change
Expand Up @@ -369,6 +369,8 @@ def __init__(
if update_on_instantiation:
self.update()

self._working_directory = None

@property
def _input_args(self):
return inspect.signature(self.node_function).parameters
Expand Down Expand Up @@ -560,6 +562,19 @@ def to_dict(self):
"signals": self.signals.to_dict(),
}

@property
def working_directory(self):
if self._working_directory is None:
if self.parent is None:
raise ValueError(
"working directory is available only if the node is"
" attached to a workflow"
)
self._working_directory = self.parent.working_directory.create_subdirectory(
self.label
)
return self._working_directory


class FastNode(Node):
"""
Expand Down
8 changes: 8 additions & 0 deletions pyiron_contrib/workflow/workflow.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
from pyiron_contrib.workflow.has_to_dict import HasToDict
from pyiron_contrib.workflow.node import Node, node, fast_node, single_value_node
from pyiron_contrib.workflow.util import DotDict
from pyiron_contrib.workflow.files import DirectoryObject


class _NodeDecoratorAccess:
Expand Down Expand Up @@ -121,10 +122,17 @@ class Workflow(HasToDict, HasNodes):
def __init__(self, label: str, *nodes: Node, strict_naming=True):
super().__init__(strict_naming=strict_naming)
self.label = label
self._working_directory = None

for node in nodes:
self.add_node(node)

@property
def working_directory(self):
if self._working_directory is None:
self._working_directory = DirectoryObject(self.label)
return self._working_directory

@property
def inputs(self):
return DotDict(
Expand Down
50 changes: 50 additions & 0 deletions tests/unit/workflow/test_files.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
import unittest
from pyiron_contrib.workflow.files import DirectoryObject, FileObject
from pathlib import Path


class TestFiles(unittest.TestCase):
def setUp(cls):
cls.directory = DirectoryObject("test")

def tearDown(cls):
cls.directory.delete()

def test_directory_exists(self):
self.assertTrue(Path("test").exists() and Path("test").is_dir())

def test_write(self):
self.directory.write(file_name="test.txt", content="something")
self.assertTrue(self.directory.file_exists("test.txt"))
self.assertTrue(
"test/test.txt" in [
ff.replace("\\", "/")
for ff in self.directory.list_content()['file']
]
)
self.assertEqual(len(self.directory), 1)

def test_create_subdirectory(self):
self.directory.create_subdirectory("another_test")
self.assertTrue(Path("test/another_test").exists())

def test_path(self):
f = FileObject("test.txt", self.directory)
self.assertEqual(str(f.path).replace("\\", "/"), "test/test.txt")

def test_read_and_write(self):
f = FileObject("test.txt", self.directory)
f.write("something")
self.assertEqual(f.read(), "something")

def test_is_file(self):
f = FileObject("test.txt", self.directory)
self.assertFalse(f.is_file())
f.write("something")
self.assertTrue(f.is_file())
f.delete()
self.assertFalse(f.is_file())


if __name__ == '__main__':
unittest.main()
6 changes: 6 additions & 0 deletions tests/unit/workflow/test_node.py
Original file line number Diff line number Diff line change
Expand Up @@ -359,6 +359,12 @@ def my_node(x: int = 0, y: int = 0, z: int = 0):
msg="After the run, all three should now be waiting for updates again"
)

def test_working_directory(self):
n_f = Node(plus_one, "output")
with self.assertRaises(ValueError):
_ = n_f.working_directory
# cf. test_workflow.py for the case that it does notraise an error


if __name__ == '__main__':
unittest.main()
20 changes: 17 additions & 3 deletions tests/unit/workflow/test_workflow.py
Original file line number Diff line number Diff line change
@@ -1,16 +1,17 @@
from unittest import TestCase, skipUnless
import unittest
from sys import version_info

from pyiron_contrib.workflow.node import Node
from pyiron_contrib.workflow.workflow import Workflow
from pyiron_contrib.workflow.files import DirectoryObject


def fnc(x=0):
return x + 1


@skipUnless(version_info[0] == 3 and version_info[1] >= 10, "Only supported for 3.10+")
class TestWorkflow(TestCase):
@unittest.skipUnless(version_info[0] == 3 and version_info[1] >= 10, "Only supported for 3.10+")
class TestWorkflow(unittest.TestCase):

def test_node_addition(self):
wf = Workflow("my_workflow")
Expand Down Expand Up @@ -107,3 +108,16 @@ def plus_one(x: int = 0) -> int:
return x + 1

self.assertEqual(plus_one().outputs.y.value, 1)

def test_working_directory(self):
wf = Workflow("wf")
self.assertTrue(wf._working_directory is None)
self.assertIsInstance(wf.working_directory, DirectoryObject)
self.assertTrue(str(wf.working_directory.path).endswith(wf.label))
wf.add.Node(fnc, "output")
self.assertTrue(str(wf.fnc.working_directory.path).endswith(wf.fnc.label))
wf.working_directory.delete()

samwaseda marked this conversation as resolved.
Show resolved Hide resolved

if __name__ == '__main__':
unittest.main()