From 0253db2677234fce139b4569afffe40294c8a9a9 Mon Sep 17 00:00:00 2001 From: "Justin R. Porter" Date: Mon, 17 Apr 2017 20:49:04 -0500 Subject: [PATCH] Build client. --- client/__init__.py | 0 client/gromppery_client.py | 173 ++++++++++++++++++++++++++++++++ client/test_client.py | 81 +++++++++++++++ gromppery/urls.py | 2 +- tprs/migrations/0001_initial.py | 48 +++++++++ tprs/migrations/__init__.py | 0 tprs/test_submission.py | 9 +- 7 files changed, 309 insertions(+), 4 deletions(-) create mode 100644 client/__init__.py create mode 100644 client/gromppery_client.py create mode 100644 client/test_client.py create mode 100644 tprs/migrations/0001_initial.py create mode 100644 tprs/migrations/__init__.py diff --git a/client/__init__.py b/client/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/client/gromppery_client.py b/client/gromppery_client.py new file mode 100644 index 0000000..2ced9fd --- /dev/null +++ b/client/gromppery_client.py @@ -0,0 +1,173 @@ +import sys +import os +import argparse +import json +import datetime +import random +import subprocess +import hashlib +import itertools +import platform + +import requests + + +def process_command_line(argv): + '''Parse the command line and do a first-pass on processing them into a + format appropriate for the rest of the script.''' + + parser = argparse.ArgumentParser(formatter_class=argparse. + ArgumentDefaultsHelpFormatter) + + parser.add_argument( + "--gromppery", required=True, + help="The URL and port where the gromppery can be found.") + parser.add_argument( + "--scratch", required=True, + help="The directory to attach to and work in.") + parser.add_argument( + "--nt", type=int, help="--nt to pass to gmx mdrun") + parser.add_argument( + "--protein", default=None, + help="Always choose this protein from the gromppery.") + parser.add_argument( + "--iterations", default=None, type=int, + help="Terminate after simulating this number of trajectories.") + + args = parser.parse_args(argv[1:]) + + if args.iterations is None: + args.iterations = itertools.count() + else: + args.iterations = range(args.iterations) + + return args + + +def get_tpr_manifest(gromppery): + '''Connect to the gromppery and get the manifest of availiable tpr + tags. + ''' + + url = '/'.join([gromppery, 'tprs.json']) + + tag_list = json.loads(requests.get(url).content.decode('utf-8')) + + return tag_list + + +def get_work(gromppery, tag): + '''Connect to the gromppery and download a the specified tpr. + ''' + + url = '/'.join([gromppery, 'tprs', tag+'.tpr']) + r = requests.get(url) + + assert r.status_code == 200, \ + "Status on get_work to %s was %s" % (url, r.status_code) + + return r.content + + +def simulate(tpr_fname, nt=None): + + base_name = tpr_fname.rstrip('.tpr') + + files = { + 'xtc': base_name+'.xtc', + 'edr': base_name+'.edr', + 'log': base_name+'.log', + 'cpt': base_name+'.cpt', + 'gro': base_name+'.gro', + 'tpr': tpr_fname + } + + mdrun_call = map(str, [ + 'gmx', 'mdrun', + '-s', files['tpr'], + '-x', files['xtc'], + '-e', files['edr'], + '-g', files['log'], + '-cpo', files['cpt'], + '-c', files['gro'], + '-v']) + + if nt is not None: + mdrun_call.extend(['-nt', nt]) + + p = subprocess.check_output(mdrun_call) + + # p.wait() + # if p.poll() != 0: + # std, err = p.communicate() + + return files + + +def submit_work(gromppery, tag, files): + + url = '/'.join([gromppery, 'tprs', tag, 'submit/']) + print(url) + + r = requests.post( + url, + data={'hostname': platform.node()}, + files={t: open(files[t], 'rb') for t + in ['xtc', 'cpt', 'gro', 'log', 'edr', 'tpr']}) + + # assert r.status_code == 201, r + try: + r.raise_for_status() + except: + with open('tmp.html', 'wb') as f: + f.write(r.content) + raise + + +def work(gromppery, scratch, protein=None): + '''The main logic of the program. Downloads, runs and submits a + random tpr from the gromppery. + ''' + + if protein is None: + tag = random.choice(get_tpr_manifest(gromppery)) + else: + tag = protein + + tprname = os.path.join(scratch, tag+'.tpr') + with open(tprname, 'wb') as f: + f.write(get_work(gromppery, tag)) + + workfiles = simulate(f.name) + submit_work(gromppery, tag, workfiles) + + +def main(argv=None): + args = process_command_line(argv) + + for i in args.iterations: + dirtries = 10 + for i in range(dirtries): + md5 = hashlib.md5(str(datetime.datetime.now().timestamp()) \ + .encode('utf-8')).hexdigest()[0:4] + dirname = os.path.join( + args.scratch, + str(datetime.datetime.now().date()) + '-' + md5) + + try: + os.mkdir(dirname) + except FileExistsError as e: + if i < dirtries: + continue + else: + raise e + else: + break + + work(args.gromppery, dirname, args.protein) + print("Finished", dirname) + + return 0 + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/client/test_client.py b/client/test_client.py new file mode 100644 index 0000000..586db88 --- /dev/null +++ b/client/test_client.py @@ -0,0 +1,81 @@ +import os +import shutil +import tempfile + +from django.conf import settings +from django.test import override_settings +from django.contrib.staticfiles.testing import StaticLiveServerTestCase + +from tprs.models import Project, Submission +from . import gromppery_client as client + + +@override_settings(MEDIA_ROOT=os.path.join(settings.BASE_DIR, 'test-media')) +class ClientTests(StaticLiveServerTestCase): + + def setUp(self): + shutil.copytree( + os.path.join(settings.BASE_DIR, 'testdata'), + os.path.join(settings.MEDIA_ROOT, 'testdata')) + + short_mdp = os.path.join(settings.MEDIA_ROOT, 'testdata', 'short.mdp') + with open(short_mdp, 'w') as mdp: + with open('testdata/plcg_sh2_wt.mdp', 'r') as f: + for line in f.readlines(): + if 'nsteps' in line.split(): + mdp.write('nsteps = 500 ; .01 ns') + elif 'nstxtcout' in line.split(): + mdp.write('nstxtcout = 25 ; 10 ps') + elif 'nstenergy' in line.split(): + mdp.write('nstenergy = 25 ; 10 ps') + else: + mdp.write(line) + + self.project = Project.objects.create( + name='plcg_sh2_wt', + gro='testdata/plcg_sh2_wt.gro', + mdp=short_mdp, + top='testdata/plcg_sh2_wt.top' + ) + + self.scratchpath = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(settings.MEDIA_ROOT) + shutil.rmtree(self.scratchpath) + + def test_submit(self): + + submission_dir = os.path.join( + settings.BASE_DIR, 'testdata', 'submission') + + files = { + 'xtc': os.path.join(submission_dir, 'plcg_sh2_wt.xtc'), + 'edr': os.path.join(submission_dir, 'plcg_sh2_wt.edr'), + 'log': os.path.join(submission_dir, 'plcg_sh2_wt.log'), + 'cpt': os.path.join(submission_dir, 'plcg_sh2_wt.cpt'), + 'gro': os.path.join(submission_dir, 'plcg_sh2_wt.gro'), + 'tpr': os.path.join(settings.BASE_DIR, 'testdata', + 'plcg_sh2_wt.tpr') + } + + client.submit_work( + self.live_server_url + '/api', + tag=self.project.name, + files=files) + + self.assertEqual(Submission.objects.count(), 1) + + sub = Submission.objects.first() + for ftype, testfile in files.items(): + self.assertEqual(getattr(sub, ftype).read(), + open(testfile, 'rb').read()) + + def test_run(self): + + client.main([ + 'gromppery_client.py', + '--protein', self.project.name, + '--scratch', self.scratchpath, + '--iterations', '1', + '--gromppery', self.live_server_url + '/api']) diff --git a/gromppery/urls.py b/gromppery/urls.py index 039daf4..918d2b9 100644 --- a/gromppery/urls.py +++ b/gromppery/urls.py @@ -25,7 +25,7 @@ urlpatterns = [ url(r'^admin/', admin.site.urls), - url(r'^tprs/(?P[\w-]+).tpr$', views.tpr, name='tpr-generate'), + url(r'^api/tprs/(?P[\w-]+).tpr$', views.tpr, name='tpr-generate'), url(r'^api/', include(router.urls)), url(r'^api-auth/', include('rest_framework.urls', namespace='rest_framework')) diff --git a/tprs/migrations/0001_initial.py b/tprs/migrations/0001_initial.py new file mode 100644 index 0000000..a5978d0 --- /dev/null +++ b/tprs/migrations/0001_initial.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.10.6 on 2017-04-16 20:49 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='Project', + fields=[ + ('name', models.CharField(max_length=200, primary_key=True, serialize=False)), + ('top', models.FileField(upload_to='projects/top')), + ('mdp', models.FileField(upload_to='projects/mdp')), + ('gro', models.FileField(upload_to='projects/gro')), + ('created', models.DateTimeField(auto_now_add=True)), + ], + options={ + 'ordering': ('created',), + }, + ), + migrations.CreateModel( + name='Submission', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('xtc', models.FileField(upload_to='submissions')), + ('edr', models.FileField(upload_to='submissions')), + ('tpr', models.FileField(upload_to='submissions')), + ('gro', models.FileField(upload_to='submissions')), + ('log', models.FileField(upload_to='submissions')), + ('cpt', models.FileField(upload_to='submissions')), + ('created', models.DateTimeField(auto_now_add=True)), + ('hostname', models.CharField(help_text='Name of the host that completed this WU', max_length=200)), + ('project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='tprs.Project')), + ], + options={ + 'ordering': ('created',), + }, + ), + ] diff --git a/tprs/migrations/__init__.py b/tprs/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tprs/test_submission.py b/tprs/test_submission.py index 13020af..7a3ad50 100644 --- a/tprs/test_submission.py +++ b/tprs/test_submission.py @@ -9,6 +9,7 @@ from rest_framework import status from rest_framework.test import APITestCase +from .seralizers import valid_xtc from .models import Project, Submission @@ -52,9 +53,11 @@ def test_submit_project(self): self.assertEqual(response.status_code, status.HTTP_201_CREATED) self.assertEqual(Submission.objects.count(), 1) - self.assertEqual(Submission.objects.first().project.pk, 'plcg_sh2_wt') - self.assertEqual(Submission.objects.first().hostname, - self.good_data['hostname']) + + submission = Submission.objects.first() + + self.assertEqual(submission.project.pk, 'plcg_sh2_wt') + self.assertEqual(submission.hostname, self.good_data['hostname']) def test_submit_bogus_tpr(self):