Build client.

client/gromppery_client.py

@@ -0,0 +1,173 @@
+import sys
+import os
+import argparse
+import json
+import datetime
+import random
+import subprocess
+import hashlib
+import itertools
+import platform
+
+import requests
+
+
+def process_command_line(argv):
+    '''Parse the command line and do a first-pass on processing them into a
+    format appropriate for the rest of the script.'''
+
+    parser = argparse.ArgumentParser(formatter_class=argparse.
+                                     ArgumentDefaultsHelpFormatter)
+
+    parser.add_argument(
+        "--gromppery", required=True,
+        help="The URL and port where the gromppery can be found.")
+    parser.add_argument(
+        "--scratch", required=True,
+        help="The directory to attach to and work in.")
+    parser.add_argument(
+        "--nt", type=int, help="--nt to pass to gmx mdrun")
+    parser.add_argument(
+        "--protein", default=None,
+        help="Always choose this protein from the gromppery.")
+    parser.add_argument(
+        "--iterations", default=None, type=int,
+        help="Terminate after simulating this number of trajectories.")
+
+    args = parser.parse_args(argv[1:])
+
+    if args.iterations is None:
+        args.iterations = itertools.count()
+    else:
+        args.iterations = range(args.iterations)
+
+    return args
+
+
+def get_tpr_manifest(gromppery):
+    '''Connect to the gromppery and get the manifest of availiable tpr
+    tags.
+    '''
+
+    url = '/'.join([gromppery, 'tprs.json'])
+
+    tag_list = json.loads(requests.get(url).content.decode('utf-8'))
+
+    return tag_list
+
+
+def get_work(gromppery, tag):
+    '''Connect to the gromppery and download a the specified tpr.
+    '''
+
+    url = '/'.join([gromppery, 'tprs', tag+'.tpr'])
+    r = requests.get(url)
+
+    assert r.status_code == 200, \
+        "Status on get_work to %s was %s" % (url, r.status_code)
+
+    return r.content
+
+
+def simulate(tpr_fname, nt=None):
+
+    base_name = tpr_fname.rstrip('.tpr')
+
+    files = {
+        'xtc': base_name+'.xtc',
+        'edr': base_name+'.edr',
+        'log': base_name+'.log',
+        'cpt': base_name+'.cpt',
+        'gro': base_name+'.gro',
+        'tpr': tpr_fname
+    }
+
+    mdrun_call = map(str, [
+        'gmx', 'mdrun',
+        '-s', files['tpr'],
+        '-x', files['xtc'],
+        '-e', files['edr'],
+        '-g', files['log'],
+        '-cpo', files['cpt'],
+        '-c', files['gro'],
+        '-v'])
+
+    if nt is not None:
+        mdrun_call.extend(['-nt', nt])
+
+    p = subprocess.check_output(mdrun_call)
+
+    # p.wait()
+    # if p.poll() != 0:
+    #     std, err = p.communicate()
+
+    return files
+
+
+def submit_work(gromppery, tag, files):
+
+    url = '/'.join([gromppery, 'tprs', tag, 'submit/'])
+    print(url)
+
+    r = requests.post(
+        url,
+        data={'hostname': platform.node()},
+        files={t: open(files[t], 'rb') for t
+               in ['xtc', 'cpt', 'gro', 'log', 'edr', 'tpr']})
+
+    # assert r.status_code == 201, r
+    try:
+        r.raise_for_status()
+    except:
+        with open('tmp.html', 'wb') as f:
+            f.write(r.content)
+        raise
+
+
+def work(gromppery, scratch, protein=None):
+    '''The main logic of the program. Downloads, runs and submits a
+    random tpr from the gromppery.
+    '''
+
+    if protein is None:
+        tag = random.choice(get_tpr_manifest(gromppery))
+    else:
+        tag = protein
+
+    tprname = os.path.join(scratch, tag+'.tpr')
+    with open(tprname, 'wb') as f:
+        f.write(get_work(gromppery, tag))
+
+    workfiles = simulate(f.name)
+    submit_work(gromppery, tag, workfiles)
+
+
+def main(argv=None):
+    args = process_command_line(argv)
+
+    for i in args.iterations:
+        dirtries = 10
+        for i in range(dirtries):
+            md5 = hashlib.md5(str(datetime.datetime.now().timestamp()) \
+                .encode('utf-8')).hexdigest()[0:4]
+            dirname = os.path.join(
+                args.scratch,
+                str(datetime.datetime.now().date()) + '-' + md5)
+
+            try:
+                os.mkdir(dirname)
+            except FileExistsError as e:
+                if i < dirtries:
+                    continue
+                else:
+                    raise e
+            else:
+                break
+
+        work(args.gromppery, dirname, args.protein)
+        print("Finished", dirname)
+
+    return 0
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))

client/test_client.py

@@ -0,0 +1,81 @@
+import os
+import shutil
+import tempfile
+
+from django.conf import settings
+from django.test import override_settings
+from django.contrib.staticfiles.testing import StaticLiveServerTestCase
+
+from tprs.models import Project, Submission
+from . import gromppery_client as client
+
+
+@override_settings(MEDIA_ROOT=os.path.join(settings.BASE_DIR, 'test-media'))
+class ClientTests(StaticLiveServerTestCase):
+
+    def setUp(self):
+        shutil.copytree(
+            os.path.join(settings.BASE_DIR, 'testdata'),
+            os.path.join(settings.MEDIA_ROOT, 'testdata'))
+
+        short_mdp = os.path.join(settings.MEDIA_ROOT, 'testdata', 'short.mdp')
+        with open(short_mdp, 'w') as mdp:
+            with open('testdata/plcg_sh2_wt.mdp', 'r') as f:
+                for line in f.readlines():
+                    if 'nsteps' in line.split():
+                        mdp.write('nsteps = 500 ; .01 ns')
+                    elif 'nstxtcout' in line.split():
+                        mdp.write('nstxtcout = 25   ; 10 ps')
+                    elif 'nstenergy' in line.split():
+                        mdp.write('nstenergy = 25   ; 10 ps')
+                    else:
+                        mdp.write(line)
+
+        self.project = Project.objects.create(
+            name='plcg_sh2_wt',
+            gro='testdata/plcg_sh2_wt.gro',
+            mdp=short_mdp,
+            top='testdata/plcg_sh2_wt.top'
+            )
+
+        self.scratchpath = tempfile.mkdtemp()
+
+    def tearDown(self):
+        shutil.rmtree(settings.MEDIA_ROOT)
+        shutil.rmtree(self.scratchpath)
+
+    def test_submit(self):
+
+        submission_dir = os.path.join(
+            settings.BASE_DIR, 'testdata', 'submission')
+
+        files = {
+            'xtc': os.path.join(submission_dir, 'plcg_sh2_wt.xtc'),
+            'edr': os.path.join(submission_dir, 'plcg_sh2_wt.edr'),
+            'log': os.path.join(submission_dir, 'plcg_sh2_wt.log'),
+            'cpt': os.path.join(submission_dir, 'plcg_sh2_wt.cpt'),
+            'gro': os.path.join(submission_dir, 'plcg_sh2_wt.gro'),
+            'tpr': os.path.join(settings.BASE_DIR, 'testdata',
+                                'plcg_sh2_wt.tpr')
+        }
+
+        client.submit_work(
+            self.live_server_url + '/api',
+            tag=self.project.name,
+            files=files)
+
+        self.assertEqual(Submission.objects.count(), 1)
+
+        sub = Submission.objects.first()
+        for ftype, testfile in files.items():
+            self.assertEqual(getattr(sub, ftype).read(),
+                             open(testfile, 'rb').read())
+
+    def test_run(self):
+
+        client.main([
+            'gromppery_client.py',
+            '--protein', self.project.name,
+            '--scratch', self.scratchpath,
+            '--iterations', '1',
+            '--gromppery', self.live_server_url + '/api'])

gromppery/urls.py

@@ -25,7 +25,7 @@
 
 urlpatterns = [
     url(r'^admin/', admin.site.urls),
-    url(r'^tprs/(?P<protein>[\w-]+).tpr$', views.tpr, name='tpr-generate'),
+    url(r'^api/tprs/(?P<protein>[\w-]+).tpr$', views.tpr, name='tpr-generate'),
     url(r'^api/', include(router.urls)),
     url(r'^api-auth/', include('rest_framework.urls',
                                namespace='rest_framework'))

tprs/migrations/0001_initial.py

@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.10.6 on 2017-04-16 20:49
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Project',
+            fields=[
+                ('name', models.CharField(max_length=200, primary_key=True, serialize=False)),
+                ('top', models.FileField(upload_to='projects/top')),
+                ('mdp', models.FileField(upload_to='projects/mdp')),
+                ('gro', models.FileField(upload_to='projects/gro')),
+                ('created', models.DateTimeField(auto_now_add=True)),
+            ],
+            options={
+                'ordering': ('created',),
+            },
+        ),
+        migrations.CreateModel(
+            name='Submission',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('xtc', models.FileField(upload_to='submissions')),
+                ('edr', models.FileField(upload_to='submissions')),
+                ('tpr', models.FileField(upload_to='submissions')),
+                ('gro', models.FileField(upload_to='submissions')),
+                ('log', models.FileField(upload_to='submissions')),
+                ('cpt', models.FileField(upload_to='submissions')),
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('hostname', models.CharField(help_text='Name of the host that completed this WU', max_length=200)),
+                ('project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='tprs.Project')),
+            ],
+            options={
+                'ordering': ('created',),
+            },
+        ),
+    ]

tprs/test_submission.py

@@ -9,6 +9,7 @@
 from rest_framework import status
 from rest_framework.test import APITestCase
 
+from .seralizers import valid_xtc
 from .models import Project, Submission
 
 
@@ -52,9 +53,11 @@ def test_submit_project(self):
 
         self.assertEqual(response.status_code, status.HTTP_201_CREATED)
         self.assertEqual(Submission.objects.count(), 1)
-        self.assertEqual(Submission.objects.first().project.pk, 'plcg_sh2_wt')
-        self.assertEqual(Submission.objects.first().hostname,
-                         self.good_data['hostname'])
+
+        submission = Submission.objects.first()
+
+        self.assertEqual(submission.project.pk, 'plcg_sh2_wt')
+        self.assertEqual(submission.hostname, self.good_data['hostname'])
 
     def test_submit_bogus_tpr(self):