Skip to content

Commit

Permalink
MIT_PV lab csv can be generated, nasty quick fixes and final csv is i…
Browse files Browse the repository at this point in the history
…mproperly formatted, but this is working
  • Loading branch information
ipendlet committed Jul 16, 2019
1 parent a7c0097 commit 4f2279e
Show file tree
Hide file tree
Showing 8 changed files with 237 additions and 64 deletions.
74 changes: 50 additions & 24 deletions expworkup/createjson.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,20 +23,40 @@ def Expdata(DatFile):
return(lines)
## File processing for the experimental JSON to convert to the final form (header of the script)

def Robo(robotfile):
def Robo(robotfile, robotfile1):
#o the file handling for the robot.xls file and return a JSON object
robot_dict = pd.read_excel(open(robotfile, 'rb'), header=[0], sheet_name=0)
reagentlist = []
for header in robot_dict.columns:
if 'Reagent' in header and "ul" in header:
reagentlist.append(header)
rnum = len(reagentlist)
robo_df = pd.read_excel(open(robotfile,'rb'), sheet_name=0,usecols=range(0,rnum+2))
robo_df_2 = pd.read_excel(open(robotfile,'rb'), sheet_name=0,usecols=[rnum+2,rnum+3]).dropna()
robo_df_3 = pd.read_excel(open(robotfile,'rb'), sheet_name=0,usecols=[rnum+4,rnum+5,rnum+6,rnum+7]).dropna()
robo_dump=json.dumps(robo_df.values.tolist())
robo_dump2=json.dumps(robo_df_2.values.tolist())
robo_dump3=json.dumps(robo_df_3.values.tolist())
# ooooh weeee that is some nasty code below, just looking for the right name
try:
robot_dict = pd.read_excel(open(robotfile, 'rb'), header=[0], sheet_name=0)
reagentlist = []
for header in robot_dict.columns:
if 'Reagent' in header and "ul" in header:
reagentlist.append(header)
rnum = len(reagentlist)
robo_df = pd.read_excel(open(robotfile,'rb'), sheet_name=0,usecols=range(0,rnum+2))
robo_df_2 = pd.read_excel(open(robotfile,'rb'), sheet_name=0,usecols=[rnum+2,rnum+3]).dropna()
robo_df_3 = pd.read_excel(open(robotfile,'rb'), sheet_name=0,usecols=[rnum+4,rnum+5,rnum+6,rnum+7]).dropna()
robo_dump=json.dumps(robo_df.values.tolist())
robo_dump2=json.dumps(robo_df_2.values.tolist())
robo_dump3=json.dumps(robo_df_3.values.tolist())
except OSError:
try:
robot_dict = pd.read_excel(open(robotfile1, 'rb'), header=[0], sheet_name=0)
robotfile = robotfile1
reagentlist = []
for header in robot_dict.columns:
if 'Precursor' in header and "ul" in header:
reagentlist.append(header)
rnum = len(reagentlist)
robo_df = pd.read_excel(open(robotfile,'rb'), sheet_name=0,usecols=range(0,rnum+3))
robo_df_2 = pd.read_excel(open(robotfile,'rb'), sheet_name=0,usecols=[rnum+3,rnum+4]).dropna()
robo_df_3 = pd.read_excel(open(robotfile,'rb'), sheet_name=0,usecols=[rnum+5,rnum+6,rnum+7,rnum+8]).dropna()
robo_dump=json.dumps(robo_df.values.tolist())
robo_dump2=json.dumps(robo_df_2.values.tolist())
robo_dump3=json.dumps(robo_df_3.values.tolist())
except OSError:
modlog.error("Failed to find correct experiment specification file, severe exit error")
sys.exit()
return(robo_dump, robo_dump2, robo_dump3)

def Crys(crysfile):
Expand All @@ -46,25 +66,31 @@ def Crys(crysfile):
:param crysfile: tabular file (.csv) used to contain experiment data
'''
headers = crysfile.pop(0)
crys_df = pd.DataFrame(crysfile, columns=headers)
crys_df_curated = crys_df[['Concatenated Vial site',
'Crystal Score',
'Bulk Actual Temp (C)',
'modelname',
'participantname',
'notes']]
crys_list = crys_df_curated.values.tolist()
crys_dump = json.dumps(crys_list)
return(crys_dump)
crys_df_curated = pd.DataFrame(crysfile, columns=headers)
# crys_df_curated = crys_df[['Concatenated Vial site',
# 'Crystal Score',
# 'Bulk Actual Temp (C)',
# 'modelname',
# 'participantname',
# 'notes']]
# crys_list = crys_df_curated.values.tolist()
# crys_dump = json.dumps(crys_list)
out_json = crys_df_curated.to_json(orient='records')
return(out_json)

def genthejson(Outfile, workdir, opfolder, drive_data):
## Do all of the file handling for a particular run and assemble the JSON, return the completed JSON file object
## and location for sorting and final comparison
Crysfile=drive_data

Expdatafile=workdir+opfolder+'_ExpDataEntry.json'
Expdatafile1=workdir+opfolder+'_preparation_interface.json'

Robofile=workdir+opfolder+'_RobotInput.xls'
Robofile1=workdir+opfolder+'_ExperimentSpecification.xls'

exp_return=Expdata(Expdatafile)
robo_return=Robo(Robofile)
robo_return=Robo(Robofile, Robofile1)
crys_return=Crys(Crysfile)
print(exp_return, file=Outfile)
print('\t},', file=Outfile)
Expand Down
120 changes: 104 additions & 16 deletions expworkup/devconfig.py
Original file line number Diff line number Diff line change
@@ -1,21 +1,109 @@
# report devconfig.py
import platform
import os
import sys

cwd = os.getcwd()

#settings = {
# "notes": {
# "ExperimentalSummary": "Stock solution preparation different than workflow 1 starting in 2018. Grind up PbI2 into a fine powder and dissolve in GBL stirring at 60C for 30 minute. Dissolve amine into GBL and heat until dissolved, combine reagents to form the primary stock solution at the desired ratio. Add amine solution to PbI2 GBL solution and stir for 45 minute then filter stock solution using PTFE 0.45 \u03bcm filter. ",
# "note1": "Difference in workflow is not captured by this reaction JSON file. Specifically the operational details of preparing the stock solution.",
# "note2": "null"
# },
# debug_dictionary = {}
# mit_dictionary = {
# 'experimental_observation_interface' {
# "Transparent?": "_rxn_transparent"
# },
# 'target_data_folder': '<enter UID> '
# }
#}
#
#######################################
# version control todo: ian where does this get used? Not sure if we should have to manually do this
RoboVersion = 2.5

SUPPORTED_LABS = ['LBL', 'HC', 'MIT_PVLab', 'ECL']

#######################################
# chemistry-relevant specifications

max_robot_reagents = 7
maxreagentchemicals = 4
volspacing = 50 # reagent microliter (uL) spacing between points in the stateset

# perovskite solvent list (simple specification of what is a liquid)
# assumes only 1 liquid / reagent
solventlist = ['GBL', 'DMSO', 'DMF', 'DCM', 'CBz']

#######################################
# Lab-specific variables

lab_vars = {
'MIT_PVLab':
{
'template_folder': '1PVeVpNjnXiAuzm3Oq2q-RiiLBhKPGW53',
'targetfolder': '1tUb4GcF_tDanMjvQuPa6vj0n9RNa5IDI', # target folder for run generation
'chemsheetid': '1htERouQUD7WR2oD-8a3KhcBpadl0kWmbipG0EFDnpcI',
'chem_workbook_index': 0,
'reagentsheetid': '1htERouQUD7WR2oD-8a3KhcBpadl0kWmbipG0EFDnpcI',
'reagent_workbook_index': 1,
'reagent_interface_amount_startrow': 16,
'max_reagents': 8, # todo: discuss
'reagent_alias': 'Precursor'
},
'HC':
{
'template_folder': '131G45eK7o9ZiDb4a2yV7l2E1WVQrz16d',
'targetfolder': '11vIE3oGU77y38VRSu-OQQw2aWaNfmOHe', # target folder for new experiments
'chemsheetid': '1JgRKUH_ie87KAXsC-fRYEw_5SepjOgVt7njjQBETxEg',
'chem_workbook_index': 0,
'reagentsheetid': '1JgRKUH_ie87KAXsC-fRYEw_5SepjOgVt7njjQBETxEg',
'reagent_workbook_index': 1,
'reagent_interface_amount_startrow': 15,
'reagent_alias': 'Reagent'
},
'LBL':
{
'template_folder': '131G45eK7o9ZiDb4a2yV7l2E1WVQrz16d',
'targetfolder': '11vIE3oGU77y38VRSu-OQQw2aWaNfmOHe', # target folder for new experiments
'chemsheetid': '1JgRKUH_ie87KAXsC-fRYEw_5SepjOgVt7njjQBETxEg',
'chem_workbook_index': 0,
'reagentsheetid': '1JgRKUH_ie87KAXsC-fRYEw_5SepjOgVt7njjQBETxEg',
'reagent_workbook_index': 1,
'reagent_interface_amount_startrow': 15,
'reagent_alias': 'Reagent',
'Robofile': '_RobotInput.xls'
}
}

#######################################
# Wolfram Kernel Management


system = platform.system()
if system == "Linux":
wolfram_kernel_path = None
from pathlib import Path
# try first path location
wolfram_kernel = Path('/usr/local/Wolfram/WolframEngine/12.0/Executables/WolframKernel')
if wolfram_kernel.is_file():
wolfram_kernel_path = "/usr/local/Wolfram/WolframEngine/12.0/Executables/WolframKernel"
# try second path location
wolfram_kernel_2 = Path('/usr/local/Wolfram/Mathematica/12.0/Executables/WolframKernel')
if wolfram_kernel_2.is_file():
wolfram_kernel_path = '/usr/local/Wolfram/Mathematica/12.0/Executables/WolframKernel'
if wolfram_kernel_path is None:
print('WolframKernel not successfully found, please correct devconfig')
import sys
sys.exit()
# Mac
elif system == "Darwin":
wolfram_kernel_path = None

######################################
# Sampler Selection


# must be 'default' or 'wolfram'
# 'wolfram' is currently experimental and unsupported
sampler = 'wolfram'

#######################################
# Laboratory file management


def labfiles(lab):
"""Returns files that need to be sent to a given laboratory"""
if lab == "LBL" or lab == "HC":
filereq = ['CrystalScoring', 'ExpDataEntry', 'metadata.json']
if lab == "MIT_PVLab":
filereq = ['observation_interface', 'preparation_interface', 'metadata.json']
if lab == 'ECL':
filereq = ['CrystalScoring', 'metadata.json']
return filereq
17 changes: 10 additions & 7 deletions expworkup/googleio.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,13 +63,16 @@ def drivedatfold(opdir):
#I have hard coded the entry to control what files we are pulling and operating on from the google drive. Users might upload similar names or
#do something I can't think of. This way we control what is loaded into the JSON
for f_sub in Exp_file_list:
if "CrystalScoring" in f_sub['title']:
Crys_dict[f['title']]=f_sub['id']
if "ExpDataEntry" in f_sub['title']:
Expdata_dict[f['title']]=f_sub['id']
if "RobotInput" in f_sub['title']:
Robo_dict[f['title']]=f_sub['id']
print('.',end='',flush=True)
if "CrystalScoring" in f_sub['title']\
or '_observation_interface' in f_sub['title']:
Crys_dict[f['title']] = f_sub['id']
if "ExpDataEntry" in f_sub['title']\
or "preparation_interface" in f_sub['title']:
Expdata_dict[f['title']] = f_sub['id']
if "RobotInput" in f_sub['title']\
or "ExperimentSpecification" in f_sub['title']:
Robo_dict[f['title']] = f_sub['id']
print('.', end='', flush=True)
print(' download complete')
return(Crys_dict, Robo_dict, Expdata_dict, dir_dict) # Returns a named list of dictionaries linked to the folder (the job jun) and the specific file's UID on gdrive. Each dictionary variable is linked to folder/run
###Returns a referenced dictionary of processed files as dictionaries {folder title SD2 ID, Gdrive UID}, the dictionary labels are thereby callable by the same key, but have different variables.. this makes sense, but likely a better way?
Expand Down
20 changes: 13 additions & 7 deletions expworkup/handlers/inchigen.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,10 @@ def GrabOrganicInchi(inchi_df, molaritydf):
pass
# pass if inchi is solvent
elif InChIKey == 'YEJRWHAVMIAJKC-UHFFFAOYSA-N' or \
InChIKey == 'ZMXDDKWLCZADIW-UHFFFAOYSA-N' or \
InChIKey == 'IAZDPXIOMUYVGZ-UHFFFAOYSA-N' or \
InChIKey == 'YMWUJEATGCHHMB-UHFFFAOYSA-N':
InChIKey == 'ZMXDDKWLCZADIW-UHFFFAOYSA-N' or\
InChIKey == 'IAZDPXIOMUYVGZ-UHFFFAOYSA-N' or\
InChIKey == 'YMWUJEATGCHHMB-UHFFFAOYSA-N' or\
InChIKey == 'MVPPADPHJFYWMZ-UHFFFAOYSA-N':
pass
# pass if lead iodide
elif InChIKey == 'RQQRAHKHDFPBMC-UHFFFAOYSA-L' or \
Expand Down Expand Up @@ -95,11 +96,16 @@ def GrabInchi(rxn_mmol_df, labels_df, inchi_df):
#pass if formic acid hard coded inchi key
if InChIKey == "BDAGIHXWWSANSR-UHFFFAOYSA-N":
pass
#pass if inchi is GBL --> 12/9/18 update now solvent ## Probably can fix this generally#
elif InChIKey =='YEJRWHAVMIAJKC-UHFFFAOYSA-N' or InChIKey=='ZMXDDKWLCZADIW-UHFFFAOYSA-N':
# pass if inchi is solvent
elif InChIKey == 'YEJRWHAVMIAJKC-UHFFFAOYSA-N' or \
InChIKey == 'ZMXDDKWLCZADIW-UHFFFAOYSA-N' or \
InChIKey == 'IAZDPXIOMUYVGZ-UHFFFAOYSA-N' or \
InChIKey == 'YMWUJEATGCHHMB-UHFFFAOYSA-N' or \
InChIKey == 'MVPPADPHJFYWMZ-UHFFFAOYSA-N':
pass
#pass if lead iodide
elif InChIKey == 'RQQRAHKHDFPBMC-UHFFFAOYSA-L' or InChIKey=='ZASWJUOMEGBQCQ-UHFFFAOYSA-L':
# pass if lead iodide
elif InChIKey == 'RQQRAHKHDFPBMC-UHFFFAOYSA-L' or \
InChIKey == 'ZASWJUOMEGBQCQ-UHFFFAOYSA-L':
pass
elif InChIKey == 'null':
pass
Expand Down
23 changes: 14 additions & 9 deletions expworkup/handlers/parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,8 @@ def reag_info(reagentdf,chemdf):
elif InChIKey == 'YEJRWHAVMIAJKC-UHFFFAOYSA-N' \
or InChIKey == 'ZMXDDKWLCZADIW-UHFFFAOYSA-N' \
or InChIKey == 'IAZDPXIOMUYVGZ-UHFFFAOYSA-N' \
or InChIKey == 'YMWUJEATGCHHMB-UHFFFAOYSA-N':
or InChIKey == 'YMWUJEATGCHHMB-UHFFFAOYSA-N' \
or InChIKey == 'MVPPADPHJFYWMZ-UHFFFAOYSA-N':
mm=(float(chemdf.loc[InChIKey,"Molecular Weight (g/mol)"]))
name=((chemdf.loc[InChIKey,"Chemical Name"]))# / float(chemdf.loc["FAH","Density (g/mL)"])
m_type='solvent'
Expand Down Expand Up @@ -272,15 +273,19 @@ def reagentparser(firstlevel, myjson, chem_df):
columnnames.append('_raw_labwareID')
well_volumes_df=pd.DataFrame(reg_value, columns=columnnames)
if reg_key == 'crys_file_data':
crys_file_data_df=pd.DataFrame(reg_value, columns=['_raw_vialsite',
'_out_crystalscore',
'_rxn_temperatureC_actual_bulk',
'_raw_modelname',
'_raw_participantname',
'_raw_notes'
])
crys_file_data_df = pd.DataFrame(reg_value)
if 'Concatenated Vial site' in crys_file_data_df.columns:
crys_file_data_df = crys_file_data_df.rename(columns = {'Concatenated Vial site': '_raw_vialsite'})
experiment_df=well_volumes_df.merge(crys_file_data_df)
#The following code aligns and normalizes the data frames
elif 'Experiment Number' in crys_file_data_df.columns:
# crys_file_data_df = crys_file_data_df.rename(columns = {'Experiment Number': 'Experiment Index'})
crys_file_data_df['_raw_vialsite'] = crys_file_data_df['Experiment Number'].astype(str)
well_volumes_df.dropna(inplace=True)
well_volumes_df['_raw_vialsite'] = well_volumes_df['_raw_vialsite'].astype(str)
# well_volumes_df.drop('_raw_vialsite', inplace=True, axis=1)
experiment_df=well_volumes_df.merge(crys_file_data_df, on='_raw_vialsite')
#The following code aligns and normalizes the data frames
experiment_df=well_volumes_df.merge(crys_file_data_df)
wellcount=(len(experiment_df.index))-1
fullrun_df=(run_df.append([run_df]*wellcount,ignore_index=True))
fullConc_df=(Conc_df.append([Conc_df]*wellcount,ignore_index=True))
Expand Down
11 changes: 10 additions & 1 deletion runme.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
from expworkup import googleio
from versiondata import export_to_repo
from tests import logger
from utils import globals

#record a detailed and organized set of the variables set by the user
def initialize(args):
Expand All @@ -31,7 +32,13 @@ def initialize(args):
# os.chdir(os.path.dirname(__file__))
parser = ap.ArgumentParser(description='Target Folder')
parser.add_argument('workdir', type=str,
help='Please include target folder')
help='Please include target folder')
# parser.add_argument('lab',
# type=str,
# choices=['LBL', 'HC', 'MIT_PVLab', 'dev'],
# help='Please include target folder',
# default='LBL'
# )
parser.add_argument('-d', '--debug', type=int, default=0,
help='Turns on testing for implementing new features to the front \
end of the code, prior to distribution through dataset')
Expand All @@ -57,6 +64,8 @@ def initialize(args):
else:
modlog.error('User MUST specify a stateset during version data repo upload preparation!')
sys.exit()

# globals.set_lab(args.lab)

initialize(args)
createjson.ExpDirOps(args.workdir, args.debug) #Run Primary JSON Creator
Expand Down
Empty file added utils/__init__.py
Empty file.
36 changes: 36 additions & 0 deletions utils/globals.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
"""A clean workaround for setting globals from the UI
Usage:
* call a setter function from runme
* you can only call a setter function once
* use the corresponding getter function to get the variable from anywhere!
* don't touch the private vars!
"""
import logging
import sys

modlog = logging.getLogger('utils.globals')

_LAB = None
_LAB_has_been_set = False


def set_lab(lab):
global _LAB, _LAB_has_been_set

if _LAB_has_been_set:
modlog.error('dev tried to run set_lab more than once')
#modlog.error('An unexpected error occurred')
sys.exit(1)

_LAB = lab
_LAB_has_been_set = True


def get_lab():
if _LAB is None:
modlog.error('get_lab called before set_lab')
#modlog.error('An unexpected error occurd')re
sys.exit(1)
return _LAB

0 comments on commit 4f2279e

Please sign in to comment.