From 6c7b21101667c252ae2ff8867c5303780bd0e654 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Thu, 29 Feb 2024 14:03:49 +0100 Subject: [PATCH 01/49] Renaming imod viewer --- imod/viewers/viewers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/imod/viewers/viewers.py b/imod/viewers/viewers.py index 794b6255..13b97867 100644 --- a/imod/viewers/viewers.py +++ b/imod/viewers/viewers.py @@ -58,6 +58,7 @@ class ImodViewer(pwviewer.Viewer): tomoObj.SetOfLandmarkModels, tomoObj.LandmarkModel ] + _name = 'Imod' def _visualize(self, obj, **kwargs): env = Plugin.getEnviron() From b02c7417c26eb03eb5ed42d65bbb91d7cc848b56 Mon Sep 17 00:00:00 2001 From: fede-pe Date: Fri, 15 Mar 2024 14:49:30 +0100 Subject: [PATCH 02/49] fix fiducila alignment: output LM generation --- imod/protocols/protocol_fiducialAlignment.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/imod/protocols/protocol_fiducialAlignment.py b/imod/protocols/protocol_fiducialAlignment.py index 0110a2d5..f261fbc0 100644 --- a/imod/protocols/protocol_fiducialAlignment.py +++ b/imod/protocols/protocol_fiducialAlignment.py @@ -689,10 +689,12 @@ def computeOutputModelsStep(self, tsObjId): prevTiltIm = 0 chainId = 0 indexFake = 0 + firstExec = True for fiducial in fiducialNoGapList: - if int(float(fiducial[2])) <= prevTiltIm: + if (int(float(fiducial[2])) <= prevTiltIm) or firstExec: chainId += 1 + firstExec = False prevTiltIm = int(float(fiducial[2])) if indexFake < len(fiducialNoGapsResidList) and fiducial[2] == fiducialNoGapsResidList[indexFake][2]: From 65e1b9d647dcae58a762d7632870afc4f9876028 Mon Sep 17 00:00:00 2001 From: JorMaister Date: Mon, 25 Mar 2024 18:52:01 +0100 Subject: [PATCH 03/49] CTF tomo correction refactored and using CTF tomo series add method getCtfTomoFromTi --- imod/protocols/protocol_base.py | 42 ++--- imod/protocols/protocol_ctfCorrection.py | 185 +++++++++++------------ imod/utils.py | 51 ++++--- 3 files changed, 137 insertions(+), 141 deletions(-) diff --git a/imod/protocols/protocol_base.py b/imod/protocols/protocol_base.py index 8148bb26..f0226394 100644 --- a/imod/protocols/protocol_base.py +++ b/imod/protocols/protocol_base.py @@ -61,6 +61,9 @@ class ProtImodBase(ProtTomoImportFiles, EMProtocol, ProtTomoBase): def __init__(self, **args): # Possible outputs (synchronize these names with the constants) + self.tsDict = None + self.binning = None + self._failedTs = [] self.TiltSeriesCoordinates = None self.FiducialModelNoGaps = None self.FiducialModelGaps = None @@ -107,7 +110,7 @@ def getTmpTSFile(self, tsId, tmpPrefix=None, suffix=".mrcs"): return os.path.join(tmpPrefix, tsId + suffix) def convertInputStep(self, tsObjId, generateAngleFile=True, - imodInterpolation=True, doSwap=False, oddEven=False): + imodInterpolation=True, doSwap=False, oddEven=False, onlyEnabled=False): """ :param tsObjId: Tilt series identifier :param generateAngleFile: Boolean(True) to generate IMOD angle file @@ -116,11 +119,13 @@ def convertInputStep(self, tsObjId, generateAngleFile=True, Pass None to cancel interpolation. :param doSwap: if applying alignment, consider swapping X/Y :param oddEven: process odd/even sets + :param onlyEnabled: flag used to include only the enabled tilt-images when generating the alignment (xf) file. """ - if isinstance(self.inputSetOfTiltSeries, SetOfTiltSeries): - ts = self.inputSetOfTiltSeries[tsObjId] - elif isinstance(self.inputSetOfTiltSeries, Pointer): - ts = self.inputSetOfTiltSeries.get()[tsObjId] + if type(tsObjId) is str: + ts = self.tsDict[tsObjId] + else: + tsSet = self.inputSetOfTiltSeries, + ts = tsSet.get()[tsObjId] if isinstance(tsSet, Pointer) else tsSet[tsObjId] tsId = ts.getTsId() @@ -153,7 +158,7 @@ def convertInputStep(self, tsObjId, generateAngleFile=True, if firstItem.hasTransform(): # Generate transformation matrices file outputTmFileName = os.path.join(tmpPrefix, firstItem.parseFileName(extension=".xf")) - utils.formatTransformFile(ts, outputTmFileName) + utils.formatTransformFile(ts, outputTmFileName, onlyEnabled=onlyEnabled) def applyNewStack(outputTsFileName, fnIn): @@ -189,9 +194,8 @@ def applyNewStack(outputTsFileName, fnIn): if generateAngleFile: """Generate angle file""" - angleFilePath = os.path.join(tmpPrefix, - firstItem.parseFileName(extension=".tlt")) - ts.generateTltFile(angleFilePath) + angleFilePath = os.path.join(tmpPrefix, firstItem.parseFileName(extension=".tlt")) + ts.generateTltFile(angleFilePath, excludeViews=True) def getBasicNewstackParams(self, ts, outputTsFileName, inputTsFileName=None, xfFile=None, firstItem=None, binning=1, doSwap=False): @@ -245,8 +249,9 @@ def getBasicNewstackParams(self, ts, outputTsFileName, inputTsFileName=None, def getOutputSetOfTiltSeries(self, inputSet, binning=1) -> SetOfTiltSeries: """ Method to generate output classes of set of tilt-series""" - if self.TiltSeries: - self.TiltSeries.enableAppend() + outputSetOfTiltSeries = getattr(self, OUTPUT_TILTSERIES_NAME, None) + if outputSetOfTiltSeries: + outputSetOfTiltSeries.enableAppend() else: outputSetOfTiltSeries = self._createSetOfTiltSeries() @@ -270,7 +275,7 @@ def getOutputSetOfTiltSeries(self, inputSet, binning=1) -> SetOfTiltSeries: self._defineOutputs(**{OUTPUT_TILTSERIES_NAME: outputSetOfTiltSeries}) self._defineSourceRelation(inputSet, outputSetOfTiltSeries) - return self.TiltSeries + return outputSetOfTiltSeries def getOutputInterpolatedSetOfTiltSeries(self, inputSet): """ Method to generate output interpolated classes of set of tilt-series""" @@ -550,21 +555,18 @@ def parseTSDefocusFile(self, inputTs, defocusFilePath, newCTFTomoSeries): newCTFTomoSeries.calculateDefocusUDeviation(defocusUTolerance=20) newCTFTomoSeries.calculateDefocusVDeviation(defocusVTolerance=20) - def createOutputFailedSet(self, tsObjId): + def createOutputFailedSet(self, ts): # Check if the tilt-series ID is in the failed tilt-series # list to add it to the set - if tsObjId in self._failedTs: - ts = self._getTiltSeries(tsObjId) + tsId = ts.getTsId() + if tsId in self._failedTs: tsSet = self._getSetOfTiltSeries() - tsId = ts.getTsId() - output = self.getOutputFailedSetOfTiltSeries(tsSet) - - newTs = TiltSeries(tsId=tsId) + newTs = ts.clone() newTs.copyInfo(ts) output.append(newTs) - for index, tiltImage in enumerate(ts): + for tiltImage in ts: newTi = TiltImage() newTi.copyInfo(tiltImage, copyId=True, copyTM=True) newTi.setAcquisition(tiltImage.getAcquisition()) diff --git a/imod/protocols/protocol_ctfCorrection.py b/imod/protocols/protocol_ctfCorrection.py index 2d730c1e..920e7492 100644 --- a/imod/protocols/protocol_ctfCorrection.py +++ b/imod/protocols/protocol_ctfCorrection.py @@ -1,6 +1,7 @@ # ***************************************************************************** # * # * Authors: Federico P. de Isidro Gomez (fp.deisidro@cnb.csic.es) [1] +# * Scipion Team (scipion@cnb.csic.es) [1] # * # * [1] Centro Nacional de Biotecnologia, CSIC, Spain # * @@ -23,16 +24,13 @@ # * e-mail address 'scipion@cnb.csic.es' # * # ***************************************************************************** - -import os - +from os.path import join from pyworkflow import BETA -from pyworkflow.object import Set +from pyworkflow.object import Set, String import pyworkflow.protocol.params as params import pyworkflow.utils.path as path -import tomo.objects as tomoObj from pwem.emlib.image import ImageHandler - +from tomo.objects import TiltSeries, TiltImage from .. import Plugin, utils from .protocol_base import ProtImodBase, EXT_MRCS_TS_ODD_NAME, EXT_MRCS_TS_EVEN_NAME @@ -47,6 +45,11 @@ class ProtImodCtfCorrection(ProtImodBase): _label = 'CTF correction' _devStatus = BETA + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.tsDict = None + self.ctfDict = None + # -------------------------- DEFINE param functions ----------------------- def _defineParams(self, form): form.addSection('Input') @@ -135,74 +138,70 @@ def _defineParams(self, form): # -------------------------- INSERT steps functions ----------------------- def _insertAllSteps(self): - self._failedTs = [] - - for ts in self.inputSetOfTiltSeries.get().iterItems(): - self._insertFunctionStep(self.convertInputStep, ts.getObjId()) - self._insertFunctionStep(self.generateDefocusFile, ts.getObjId()) - self._insertFunctionStep(self.ctfCorrection, ts.getObjId()) - self._insertFunctionStep(self.createOutputStep, ts.getObjId()) - self._insertFunctionStep(self.createOutputFailedSet, ts.getObjId()) + nonMatchingTsIds = [] + self._initialize() + for tsId in self.tsDict.keys(): # Stores the steps serializing the tsId instead of the whole ts object + matchingCtfTomoSeries = self.ctfDict.get(tsId, None) + if matchingCtfTomoSeries: + self._insertFunctionStep(self.convertInputStep, tsId) + self._insertFunctionStep(self.generateDefocusFile, tsId) + self._insertFunctionStep(self.ctfCorrection, tsId) + self._insertFunctionStep(self.createOutputStep, tsId) + self._insertFunctionStep(self.createOutputFailedSet, tsId) + else: + nonMatchingTsIds.append(tsId) self._insertFunctionStep(self.closeOutputSetsStep) + if nonMatchingTsIds: + self.matchingMsg.set(f'WARNING! No CTFTomoSeries found for the tilt-series {nonMatchingTsIds}') + self._store(self.matchingMsg) # --------------------------- STEPS functions ----------------------------- - def convertInputStep(self, tsObjId, **kwargs): + def _initialize(self): + self.matchingMsg = String() # Update the msg for each protocol execution to avoid duplicities in the summary + self.tsDict = {ts.getTsId(): ts.clone(ignoreAttrs=[]) for ts in self.inputSetOfTiltSeries.get()} + self.ctfDict = {ctf.getTsId(): ctf.clone(ignoreAttrs=[]) for ctf in self.inputSetOfCtfTomoSeries.get()} + + def convertInputStep(self, tsId, **kwargs): oddEvenFlag = self.applyToOddEven(self.inputSetOfTiltSeries.get()) # Considering swapXY is required to make tilt axis vertical - super().convertInputStep(tsObjId, doSwap=True, oddEven=oddEvenFlag) - - def tsToProcess(self, tsObjId) -> bool: - tsId = self.inputSetOfTiltSeries.get()[tsObjId].getTsId() - ctfTomoSeries = self.getCtfTomoSeriesFromTsId(tsId) - return ctfTomoSeries - - def generateDefocusFile(self, tsObjId): - if self.tsToProcess(tsObjId) is None: - return - ts = self.inputSetOfTiltSeries.get()[tsObjId] - tsId = ts.getTsId() - - self.debug(f"Generating defocus file for {tsObjId} (ObjId), {tsId} (TsId)") - + super().convertInputStep(tsId, doSwap=True, oddEven=oddEvenFlag, onlyEnabled=True) + + def generateDefocusFile(self, tsId): + ts = self.tsDict[tsId] + ctfTomoSeries = self.ctfDict[tsId] + + self.debug(f"Generating defocus file for {tsId} (ObjId), {tsId} (TsId)") + # # At this step, a tlt file has been generated considering the excluded views in the current TS. If the no. + # # tilt images is different from the no. CTFTomos, the attribute _isEnabled of the elements that are not present + # # in the TS or are disabled tilt-images will be set to False and considered when generating the defocus IMOD + # # files + # if len(ts) != len(ctfTomoSeries): + # ctfTomoSeries.updateCtfTomoEnableFromTs(ts) # Compose the defocus file path defocusFilePath = self.getDefocusFileName(ts) - """Generate defocus file""" - ctfTomoSeries = self.getCtfTomoSeriesFromTsId(tsId) utils.generateDefocusIMODFileFromObject(ctfTomoSeries, defocusFilePath, inputTiltSeries=ts) - def getDefocusFileName(self, ts): - """ Returns the path of the defocus filename based on - the tilt series and creates the folder/s""" - - tmpPrefix = self._getTmpPath(ts.getTsId()) - path.makePath(tmpPrefix) - defocusFn = ts.getFirstItem().parseFileName(extension=".defocus") - defocusFilePath = os.path.join(tmpPrefix, defocusFn) - return defocusFilePath - @ProtImodBase.tryExceptDecorator - def ctfCorrection(self, tsObjId): - if self.tsToProcess(tsObjId) is None: - return - ts = self.inputSetOfTiltSeries.get()[tsObjId] - tsId = ts.getTsId() + def ctfCorrection(self, tsId): + ts = self.tsDict[tsId] + acq = ts.getAcquisition() extraPrefix = self._getExtraPath(tsId) tmpPrefix = self._getTmpPath(tsId) firstItem = ts.getFirstItem() """Run ctfphaseflip IMOD program""" paramsCtfPhaseFlip = { - 'inputStack': os.path.join(tmpPrefix, firstItem.parseFileName()), - 'angleFile': os.path.join(tmpPrefix, firstItem.parseFileName(extension=".tlt")), - 'outputFileName': os.path.join(extraPrefix, firstItem.parseFileName()), + 'inputStack': join(tmpPrefix, firstItem.parseFileName()), + 'angleFile': join(tmpPrefix, firstItem.parseFileName(extension=".tlt")), + 'outputFileName': join(extraPrefix, firstItem.parseFileName()), 'defocusFile': self.getDefocusFileName(ts), - 'voltage': self.inputSetOfTiltSeries.get().getAcquisition().getVoltage(), - 'sphericalAberration': self.inputSetOfTiltSeries.get().getAcquisition().getSphericalAberration(), + 'voltage': acq.getVoltage(), + 'sphericalAberration': ts.getAcquisition().getSphericalAberration(), 'defocusTol': self.defocusTol.get(), - 'pixelSize': self.inputSetOfTiltSeries.get().getSamplingRate() / 10, - 'amplitudeContrast': self.inputSetOfTiltSeries.get().getAcquisition().getAmplitudeContrast(), + 'pixelSize': ts.getSamplingRate() / 10, + 'amplitudeContrast': acq.getAmplitudeContrast(), 'interpolationWidth': self.interpolationWidth.get() } @@ -222,77 +221,65 @@ def ctfCorrection(self, tsObjId): "-ActionIfGPUFails 2,2 " if ts.getFirstItem().hasTransform(): - paramsCtfPhaseFlip['xformFile'] = os.path.join(tmpPrefix, firstItem.parseFileName(extension=".xf")) + paramsCtfPhaseFlip['xformFile'] = join(tmpPrefix, firstItem.parseFileName(extension=".xf")) argsCtfPhaseFlip += "-TransformFile %(xformFile)s " Plugin.runImod(self, 'ctfphaseflip', argsCtfPhaseFlip % paramsCtfPhaseFlip) if self.applyToOddEven(ts): - #ODD + # ODD oddFn = self.getTmpTSFile(tsId, tmpPrefix=tmpPrefix, suffix=EXT_MRCS_TS_ODD_NAME) paramsCtfPhaseFlip['inputStack'] = oddFn - paramsCtfPhaseFlip['outputFileName'] = os.path.join(extraPrefix, tsId+EXT_MRCS_TS_ODD_NAME) + paramsCtfPhaseFlip['outputFileName'] = join(extraPrefix, tsId+EXT_MRCS_TS_ODD_NAME) Plugin.runImod(self, 'ctfphaseflip', argsCtfPhaseFlip % paramsCtfPhaseFlip) # EVEN evenFn = self.getTmpTSFile(tsId, tmpPrefix=tmpPrefix, suffix=EXT_MRCS_TS_EVEN_NAME) paramsCtfPhaseFlip['inputStack'] = evenFn - paramsCtfPhaseFlip['outputFileName'] = os.path.join(extraPrefix, tsId+EXT_MRCS_TS_EVEN_NAME) + paramsCtfPhaseFlip['outputFileName'] = join(extraPrefix, tsId+EXT_MRCS_TS_EVEN_NAME) Plugin.runImod(self, 'ctfphaseflip', argsCtfPhaseFlip % paramsCtfPhaseFlip) - def createOutputStep(self, tsObjId): - if self.tsToProcess(tsObjId) is None: - return - if tsObjId not in self._failedTs: - inputTs = self.inputSetOfTiltSeries.get() - output = self.getOutputSetOfTiltSeries(inputTs) - hasAlign = inputTs.getFirstItem().getFirstItem().hasTransform() - - ts = inputTs[tsObjId] - tsId = ts.getTsId() + def createOutputStep(self, tsId): + if tsId not in self._failedTs: + inTsSet = self.inputSetOfTiltSeries.get() + outputSetOfTs = self.getOutputSetOfTiltSeries(inTsSet) extraPrefix = self._getExtraPath(tsId) - newTs = tomoObj.TiltSeries(tsId=tsId) + newTs = TiltSeries(tsId=tsId) + ts = self.tsDict[tsId] newTs.copyInfo(ts) newTs.setCtfCorrected(True) newTs.setInterpolated(True) - output.append(newTs) + acq = newTs.getAcquisition() + acq.setTiltAxisAngle(0.) # 0 because TS is aligned + newTs.setAcquisition(acq) + outputSetOfTs.append(newTs) ih = ImageHandler() for index, tiltImage in enumerate(ts): - newTi = tomoObj.TiltImage() + newTi = TiltImage() newTi.copyInfo(tiltImage, copyId=True, copyTM=False) acq = tiltImage.getAcquisition() - if hasAlign: - acq.setTiltAxisAngle(0.) + acq.setTiltAxisAngle(0.) # Is interpolated newTi.setAcquisition(acq) - newTi.setLocation(index + 1, - (os.path.join(extraPrefix, - tiltImage.parseFileName()))) + newTi.setLocation(index + 1, (join(extraPrefix, tiltImage.parseFileName()))) if self.applyToOddEven(ts): - locationOdd = index + 1, (os.path.join(extraPrefix, tsId + EXT_MRCS_TS_ODD_NAME)) - locationEven = index + 1, (os.path.join(extraPrefix, tsId + EXT_MRCS_TS_EVEN_NAME)) + locationOdd = index + 1, (join(extraPrefix, tsId + EXT_MRCS_TS_ODD_NAME)) + locationEven = index + 1, (join(extraPrefix, tsId + EXT_MRCS_TS_EVEN_NAME)) newTi.setOddEven([ih.locationToXmipp(locationOdd), ih.locationToXmipp(locationEven)]) else: newTi.setOddEven([]) - newTs.append(newTi) - if hasAlign: - acq = newTs.getAcquisition() - acq.setTiltAxisAngle(0.) # 0 because TS is aligned - newTs.setAcquisition(acq) - newTs.write(properties=False) - output.update(newTs) - output.write() - self._store() + outputSetOfTs.update(newTs) + outputSetOfTs.write() + self._store(outputSetOfTs) - def createOutputFailedSet(self, tsObjId): - if self.tsToProcess(tsObjId) is None: - return - super().createOutputFailedSet(tsObjId) + def createOutputFailedSet(self, tsId): + ts = self.tsDict[tsId] + super().createOutputFailedSet(ts) def closeOutputSetsStep(self): for _, output in self.iterOutputAttributes(): @@ -301,11 +288,15 @@ def closeOutputSetsStep(self): self._store() # --------------------------- UTILS functions ----------------------------- - def getCtfTomoSeriesFromTsId(self, tsId): - for ctfTomoSeries in self.inputSetOfCtfTomoSeries.get(): - if tsId == ctfTomoSeries.getTsId(): - return ctfTomoSeries - return None + def getDefocusFileName(self, ts): + """ Returns the path of the defocus filename based on + the tilt series and creates the folder/s""" + + tmpPrefix = self._getTmpPath(ts.getTsId()) + path.makePath(tmpPrefix) + defocusFn = ts.getFirstItem().parseFileName(extension=".defocus") + defocusFilePath = join(tmpPrefix, defocusFn) + return defocusFilePath # --------------------------- INFO functions ------------------------------ def _warnings(self): @@ -328,6 +319,8 @@ def _summary(self): self.TiltSeries.getSize())) else: summary.append("Outputs are not ready yet.") + if self.matchingMsg.get(): + summary.append(self.matchingMsg.get()) return summary def _methods(self): diff --git a/imod/utils.py b/imod/utils.py index 0f764ec5..d54ed6cc 100644 --- a/imod/utils.py +++ b/imod/utils.py @@ -29,6 +29,8 @@ import logging import os +from tomo.objects import CTFTomo, TiltImage + logger = logging.getLogger(__name__) import csv import math @@ -39,7 +41,7 @@ from imod import Plugin -def formatTransformFile(ts, transformFilePath): +def formatTransformFile(ts, transformFilePath, onlyEnabled=False): """ This method takes a tilt series and the output transformation file path and creates an IMOD-based transform file in the location indicated. """ @@ -47,6 +49,8 @@ def formatTransformFile(ts, transformFilePath): tsMatrixTransformList = [] for ti in ts: + if onlyEnabled and not ti.isEnabled(): + continue transform = ti.getTransform().getMatrix().flatten() transformIMOD = ['%.7f' % transform[0], '%.7f' % transform[1], @@ -822,26 +826,23 @@ def generateDefocusIMODFileFromObject(ctfTomoSeries, defocusFilePath, with open(defocusFilePath, 'w') as f: lines = ["1\t0\t0.0\t0.0\t0.0\t3\n"] + for index, ti in enumerate(tiltSeries): + ctfTomo = ctfTomoSeries.getCtfTomoFromTi(ti) + if ctfTomo: + tiltAngle = ti.getTiltAngle() + + newLine = ("%d\t%d\t%.2f\t%.2f\t%.1f\t%.1f\t%.2f\n" % ( + index, + index, + tiltAngle, + tiltAngle, + # CONVERT DEFOCUS VALUE TO NANOMETERS (IMOD CONVENTION) + ctfTomo.getDefocusU()/10, + # CONVERT DEFOCUS VALUE TO NANOMETERS (IMOD CONVENTION) + ctfTomo.getDefocusV()/10, + ctfTomo.getDefocusAngle())) - # CtfTomoSeries is iterated inversely because IMOD set indexes - # upside down Scipion (highest index for - # the tilt-image with the highest negative angle) - for ctfTomo in ctfTomoSeries: - index = ctfTomo.getIndex().get() - - newLine = ("%d\t%d\t%.2f\t%.2f\t%.1f\t%.1f\t%.2f\n" % ( - index, - index, - tiltSeries[index].getTiltAngle(), - tiltSeries[index].getTiltAngle(), - # CONVERT DEFOCUS VALUE TO NANOMETERS (IMOD CONVENTION) - ctfTomo.getDefocusU()/10, - # CONVERT DEFOCUS VALUE TO NANOMETERS (IMOD CONVENTION) - ctfTomo.getDefocusV()/10, - ctfTomo.getDefocusAngle()) - ) - - lines.append(newLine) + lines.append(newLine) f.writelines(lines) @@ -857,7 +858,7 @@ def generateDefocusUDictionary(ctfTomoSeries): else ctfTomo.getDefocusVList() defocusInfoList = defocusInfoList.split(",") - index = ctfTomo.getIndex().get() + index = ctfTomo.getIndex() defocusUDict[index] = defocusInfoList @@ -875,7 +876,7 @@ def generateDefocusVDictionary(ctfTomoSeries): defocusInfoList = ctfTomo.getDefocusVList() defocusInfoList = defocusInfoList.split(",") - index = ctfTomo.getIndex().get() + index = ctfTomo.getIndex() defocusVDict[index] = defocusInfoList @@ -893,7 +894,7 @@ def generateDefocusAngleDictionary(ctfTomoSeries): defocusAngleList = ctfTomo.getDefocusAngleList() defocusAngleList = defocusAngleList.split(",") - index = ctfTomo.getIndex().get() + index = ctfTomo.getIndex() defocusAngleDict[index] = defocusAngleList @@ -911,7 +912,7 @@ def generatePhaseShiftDictionary(ctfTomoSeries): phaseShiftList = ctfTomo.getPhaseShiftList() phaseShiftList = phaseShiftList.split(",") - index = ctfTomo.getIndex().get() + index = ctfTomo.getIndex() phaseShiftDict[index] = phaseShiftList @@ -929,7 +930,7 @@ def generateCutOnFreqDictionary(ctfTomoSeries): cutOnFreqList = ctfTomo.getCutOnFreqList() cutOnFreqList = cutOnFreqList.split(",") - index = ctfTomo.getIndex().get() + index = ctfTomo.getIndex() cutOnFreqDict[index] = cutOnFreqList From c203a251ca9a12959a85de1d52668d4ecf1f142c Mon Sep 17 00:00:00 2001 From: JorMaister Date: Tue, 26 Mar 2024 12:23:31 +0100 Subject: [PATCH 04/49] - Protocol CTF correction: * Now it can deal with excluded views in the CTF and/or in the TS. * The xf, defocus, and tlt files are generated inside each TS extra sub-directory. Some code cleaning --- imod/protocols/protocol_base.py | 106 ++++++++++++----------- imod/protocols/protocol_ctfCorrection.py | 11 ++- imod/utils.py | 16 ++-- 3 files changed, 68 insertions(+), 65 deletions(-) diff --git a/imod/protocols/protocol_base.py b/imod/protocols/protocol_base.py index f0226394..881b30d4 100644 --- a/imod/protocols/protocol_base.py +++ b/imod/protocols/protocol_base.py @@ -23,7 +23,7 @@ # * e-mail address 'scipion@cnb.csic.es' # * # ***************************************************************************** - +import logging import os from pyworkflow.object import Set, CsvList, Pointer @@ -33,11 +33,13 @@ from pwem.protocols import EMProtocol from tomo.protocols.protocol_base import ProtTomoBase, ProtTomoImportFiles from tomo.objects import (SetOfTiltSeries, SetOfTomograms, SetOfCTFTomoSeries, - CTFTomo, SetOfTiltSeriesCoordinates, TiltSeries, - TiltImage) - + CTFTomo, SetOfTiltSeriesCoordinates, TiltImage) from .. import Plugin, utils + +logger = logging.getLogger(__name__) + + OUTPUT_TS_COORDINATES_NAME = "TiltSeriesCoordinates" OUTPUT_FIDUCIAL_NO_GAPS_NAME = "FiducialModelNoGaps" OUTPUT_FIDUCIAL_GAPS_NAME = "FiducialModelGaps" @@ -62,7 +64,7 @@ def __init__(self, **args): # Possible outputs (synchronize these names with the constants) self.tsDict = None - self.binning = None + self.binning = 1 self._failedTs = [] self.TiltSeriesCoordinates = None self.FiducialModelNoGaps = None @@ -78,7 +80,8 @@ def __init__(self, **args): @classmethod def worksInStreaming(cls): - """ So far none of them work in streaming. Since this inherits from the import they were considered as "streamers". """ + """ So far none of them work in streaming. Since this inherits from the import they were considered as + "streamers".""" return False def defineExecutionPararell(self): @@ -103,6 +106,7 @@ def wrapper(self, tsId, *args): self._failedTs.append(tsId) return wrapper + def getTmpTSFile(self, tsId, tmpPrefix=None, suffix=".mrcs"): if tmpPrefix is None: tmpPrefix = self._getTmpPath(tsId) @@ -128,58 +132,49 @@ def convertInputStep(self, tsObjId, generateAngleFile=True, ts = tsSet.get()[tsObjId] if isinstance(tsSet, Pointer) else tsSet[tsObjId] tsId = ts.getTsId() - extraPrefix = self._getExtraPath(tsId) tmpPrefix = self._getTmpPath(tsId) - path.makePath(tmpPrefix) path.makePath(extraPrefix) - firstItem = ts.getFirstItem() + firstTi = ts.getFirstItem() + inTsFileName = firstTi.getFileName() + outputTsFileName = os.path.join(tmpPrefix, firstTi.parseFileName()) - outputTsFileName = os.path.join(tmpPrefix, firstItem.parseFileName()) - - if oddEven: - fnOdd = ts.getOddFileName() - fnEven = ts.getEvenFileName() - - outputOddTsFileName = self.getTmpTSFile(tsId, tmpPrefix=tmpPrefix, suffix=EXT_MRCS_TS_ODD_NAME) - outputEvenTsFileName = self.getTmpTSFile(tsId, tmpPrefix=tmpPrefix, suffix=EXT_MRCS_TS_EVEN_NAME) - - # .. Interpolation cancelled + # Interpolation if imodInterpolation is None: - self.info("Tilt series %s linked." % tsId) - path.createLink(firstItem.getFileName(), outputTsFileName) - + logger.info("Tilt series %s linked." % tsId) + path.createLink(inTsFileName, outputTsFileName) elif imodInterpolation: - """Apply the transformation form the input tilt-series""" + logger.info("Apply the transformation form the input tilt-series") + + # Odd / Even + outputOddTsFileName = None + outputEvenTsFileName = None + fnOdd = None + fnEven = None + if oddEven: + fnOdd = ts.getOddFileName() + fnEven = ts.getEvenFileName() + outputOddTsFileName = self.getTmpTSFile(tsId, tmpPrefix=tmpPrefix, suffix=EXT_MRCS_TS_ODD_NAME) + outputEvenTsFileName = self.getTmpTSFile(tsId, tmpPrefix=tmpPrefix, suffix=EXT_MRCS_TS_EVEN_NAME) # Use IMOD newstack interpolation - if firstItem.hasTransform(): - # Generate transformation matrices file - outputTmFileName = os.path.join(tmpPrefix, firstItem.parseFileName(extension=".xf")) - utils.formatTransformFile(ts, outputTmFileName, onlyEnabled=onlyEnabled) - - def applyNewStack(outputTsFileName, fnIn): - - argsAlignment, paramsAlignment = self.getBasicNewstackParams(ts, - outputTsFileName, - inputTsFileName=fnIn, - xfFile=outputTmFileName, - firstItem=firstItem, - doSwap=doSwap) - Plugin.runImod(self, 'newstack', argsAlignment % paramsAlignment) - - self.info("Interpolating tilt series %s with imod" % tsId) - applyNewStack(outputTsFileName, None) - + if firstTi.hasTransform(): + # Generate transformation matrices file (xf) + xfFile = os.path.join(extraPrefix, firstTi.parseFileName(extension=".xf")) + utils.formatTransformFile(ts, xfFile, onlyEnabled=onlyEnabled) + + # Generate the interpolated TS with IMOD's newstack program + logger.info("Tilt-series interpolated with IMOD [%s]" % tsId) + self.applyNewStackBasic(ts, outputTsFileName, inTsFileName, xfFile, doSwap) if oddEven: - applyNewStack(outputOddTsFileName, fnOdd) - applyNewStack(outputEvenTsFileName, fnEven) + self.applyNewStackBasic(ts, outputOddTsFileName, fnOdd, xfFile, doSwap) + self.applyNewStackBasic(ts, outputEvenTsFileName, fnEven, xfFile, doSwap) else: - self.info("Linking tilt series %s" % tsId) - path.createLink(firstItem.getFileName(), outputTsFileName) + logger.info("Tilt-series linked [%s]" % tsId) + path.createLink(firstTi.getFileName(), outputTsFileName) if oddEven: path.createLink(fnOdd, outputOddTsFileName) @@ -187,17 +182,28 @@ def applyNewStack(outputTsFileName, fnIn): # Use Xmipp interpolation via Scipion else: - self.info("Interpolating tilt series %s with emlib" % tsId) + logger.info("Tilt-series interpolated with emlib [%s]" % tsId) ts.applyTransform(outputTsFileName) - self.info("Tilt series %s available for processing at %s." % (tsId, outputTsFileName)) + logger.info("Tilt-series [%s] available for processing at %s." % (tsId, outputTsFileName)) + # Generate the tlt file if generateAngleFile: - """Generate angle file""" - angleFilePath = os.path.join(tmpPrefix, firstItem.parseFileName(extension=".tlt")) + logger.info("Generate angle file for the tilt-series [%s]" % tsId) + angleFilePath = os.path.join(extraPrefix, firstTi.parseFileName(extension=".tlt")) ts.generateTltFile(angleFilePath, excludeViews=True) - def getBasicNewstackParams(self, ts, outputTsFileName, inputTsFileName=None, + def applyNewStackBasic(self, ts, outputTsFileName, inputTsFileName, xfFile, doSwap): + argsAlignment, paramsAlignment = self.getBasicNewstackParams(ts, + outputTsFileName, + inputTsFileName=inputTsFileName, + xfFile=xfFile, + firstItem=ts.getFirstItem(), + doSwap=doSwap) + Plugin.runImod(self, 'newstack', argsAlignment % paramsAlignment) + + @staticmethod + def getBasicNewstackParams(ts, outputTsFileName, inputTsFileName=None, xfFile=None, firstItem=None, binning=1, doSwap=False): """ Returns basic newstack arguments diff --git a/imod/protocols/protocol_ctfCorrection.py b/imod/protocols/protocol_ctfCorrection.py index 920e7492..b5d3ca78 100644 --- a/imod/protocols/protocol_ctfCorrection.py +++ b/imod/protocols/protocol_ctfCorrection.py @@ -163,7 +163,6 @@ def _initialize(self): def convertInputStep(self, tsId, **kwargs): oddEvenFlag = self.applyToOddEven(self.inputSetOfTiltSeries.get()) - # Considering swapXY is required to make tilt axis vertical super().convertInputStep(tsId, doSwap=True, oddEven=oddEvenFlag, onlyEnabled=True) @@ -194,7 +193,7 @@ def ctfCorrection(self, tsId): """Run ctfphaseflip IMOD program""" paramsCtfPhaseFlip = { 'inputStack': join(tmpPrefix, firstItem.parseFileName()), - 'angleFile': join(tmpPrefix, firstItem.parseFileName(extension=".tlt")), + 'angleFile': join(extraPrefix, firstItem.parseFileName(extension=".tlt")), 'outputFileName': join(extraPrefix, firstItem.parseFileName()), 'defocusFile': self.getDefocusFileName(ts), 'voltage': acq.getVoltage(), @@ -221,7 +220,7 @@ def ctfCorrection(self, tsId): "-ActionIfGPUFails 2,2 " if ts.getFirstItem().hasTransform(): - paramsCtfPhaseFlip['xformFile'] = join(tmpPrefix, firstItem.parseFileName(extension=".xf")) + paramsCtfPhaseFlip['xformFile'] = join(extraPrefix, firstItem.parseFileName(extension=".xf")) argsCtfPhaseFlip += "-TransformFile %(xformFile)s " Plugin.runImod(self, 'ctfphaseflip', argsCtfPhaseFlip % paramsCtfPhaseFlip) @@ -292,10 +291,10 @@ def getDefocusFileName(self, ts): """ Returns the path of the defocus filename based on the tilt series and creates the folder/s""" - tmpPrefix = self._getTmpPath(ts.getTsId()) - path.makePath(tmpPrefix) + extraPrefix = self._getExtraPath(ts.getTsId()) + path.makePath(extraPrefix) defocusFn = ts.getFirstItem().parseFileName(extension=".defocus") - defocusFilePath = join(tmpPrefix, defocusFn) + defocusFilePath = join(extraPrefix, defocusFn) return defocusFilePath # --------------------------- INFO functions ------------------------------ diff --git a/imod/utils.py b/imod/utils.py index d54ed6cc..cc7011d4 100644 --- a/imod/utils.py +++ b/imod/utils.py @@ -28,19 +28,17 @@ """ import logging import os - -from tomo.objects import CTFTomo, TiltImage - -logger = logging.getLogger(__name__) import csv import math import numpy as np - import pyworkflow.object as pwobj import pyworkflow.utils as pwutils from imod import Plugin +logger = logging.getLogger(__name__) + + def formatTransformFile(ts, transformFilePath, onlyEnabled=False): """ This method takes a tilt series and the output transformation file path and creates an IMOD-based transform @@ -822,7 +820,7 @@ def generateDefocusIMODFileFromObject(ctfTomoSeries, defocusFilePath, else: # There is no information available as list (not an IMOD CTF estimation) - logger.debug("Defocus file generated form a defocus attributes.") + logger.info("Defocus file generated from defocus attributes.") with open(defocusFilePath, 'w') as f: lines = ["1\t0\t0.0\t0.0\t0.0\t3\n"] @@ -830,10 +828,10 @@ def generateDefocusIMODFileFromObject(ctfTomoSeries, defocusFilePath, ctfTomo = ctfTomoSeries.getCtfTomoFromTi(ti) if ctfTomo: tiltAngle = ti.getTiltAngle() - + ind = index + 1 newLine = ("%d\t%d\t%.2f\t%.2f\t%.1f\t%.1f\t%.2f\n" % ( - index, - index, + ind, + ind, tiltAngle, tiltAngle, # CONVERT DEFOCUS VALUE TO NANOMETERS (IMOD CONVENTION) From d078eb51edb9e1b6463ebdb18fdd1a12a84dbef8 Mon Sep 17 00:00:00 2001 From: JorMaister Date: Tue, 26 Mar 2024 12:27:20 +0100 Subject: [PATCH 05/49] Update changes, bump version --- CHANGES.txt | 7 +++++++ imod/__init__.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index 4aec7ef1..b64b7581 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,3 +1,10 @@ +3.5.0: + Users: + - Protocol CTF correction: + * Now it can deal with excluded views in the CTF and/or in the TS. + * The xf, defocus, and tlt files are generated inside each TS extra sub-directory. +3.4.1: # TODO: Someone forgot to fill this. +3.4.0: # TODO: Someone forgot to fill this. 3.3.0: - bugfix for import ctf, set missing defocus flag - move plugin-specific import CTF protocol to the core diff --git a/imod/__init__.py b/imod/__init__.py index 118f6e0e..1ee66a4f 100644 --- a/imod/__init__.py +++ b/imod/__init__.py @@ -35,7 +35,7 @@ from .constants import IMOD_HOME, ETOMO_CMD, DEFAULT_VERSION, VERSIONS, IMOD_VIEWER_BINNING -__version__ = '3.4.1' +__version__ = '3.5.0' _logo = "icon.png" _references = ['Kremer1996', 'Mastronarde2017'] From 88a0c1a596e56fc01b04e24145ba8b4feb80c536 Mon Sep 17 00:00:00 2001 From: JorMaister Date: Tue, 26 Mar 2024 18:49:19 +0100 Subject: [PATCH 06/49] Improvements --- imod/protocols/protocol_base.py | 29 +++++++++++++++--------- imod/protocols/protocol_ctfCorrection.py | 8 +------ imod/utils.py | 5 ++-- 3 files changed, 22 insertions(+), 20 deletions(-) diff --git a/imod/protocols/protocol_base.py b/imod/protocols/protocol_base.py index 881b30d4..f4e03f72 100644 --- a/imod/protocols/protocol_base.py +++ b/imod/protocols/protocol_base.py @@ -113,8 +113,7 @@ def getTmpTSFile(self, tsId, tmpPrefix=None, suffix=".mrcs"): return os.path.join(tmpPrefix, tsId + suffix) - def convertInputStep(self, tsObjId, generateAngleFile=True, - imodInterpolation=True, doSwap=False, oddEven=False, onlyEnabled=False): + def convertInputStep(self, tsObjId, generateAngleFile=True, imodInterpolation=True, doSwap=False, oddEven=False): """ :param tsObjId: Tilt series identifier :param generateAngleFile: Boolean(True) to generate IMOD angle file @@ -123,7 +122,6 @@ def convertInputStep(self, tsObjId, generateAngleFile=True, Pass None to cancel interpolation. :param doSwap: if applying alignment, consider swapping X/Y :param oddEven: process odd/even sets - :param onlyEnabled: flag used to include only the enabled tilt-images when generating the alignment (xf) file. """ if type(tsObjId) is str: ts = self.tsDict[tsObjId] @@ -163,14 +161,18 @@ def convertInputStep(self, tsObjId, generateAngleFile=True, if firstTi.hasTransform(): # Generate transformation matrices file (xf) xfFile = os.path.join(extraPrefix, firstTi.parseFileName(extension=".xf")) - utils.formatTransformFile(ts, xfFile, onlyEnabled=onlyEnabled) + utils.formatTransformFile(ts, xfFile) # Generate the interpolated TS with IMOD's newstack program logger.info("Tilt-series interpolated with IMOD [%s]" % tsId) - self.applyNewStackBasic(ts, outputTsFileName, inTsFileName, xfFile, doSwap) + tsExcludedIndices = [ti.getIndex() for ti in ts if not ti.isEnabled()] + self.applyNewStackBasic(ts, outputTsFileName, inTsFileName, xfFile, doSwap, + tsExcludedIndices=tsExcludedIndices) if oddEven: - self.applyNewStackBasic(ts, outputOddTsFileName, fnOdd, xfFile, doSwap) - self.applyNewStackBasic(ts, outputEvenTsFileName, fnEven, xfFile, doSwap) + self.applyNewStackBasic(ts, outputOddTsFileName, fnOdd, xfFile, doSwap, + tsExcludedIndices=tsExcludedIndices) + self.applyNewStackBasic(ts, outputEvenTsFileName, fnEven, xfFile, doSwap, + tsExcludedIndices=tsExcludedIndices) else: logger.info("Tilt-series linked [%s]" % tsId) @@ -193,18 +195,19 @@ def convertInputStep(self, tsObjId, generateAngleFile=True, angleFilePath = os.path.join(extraPrefix, firstTi.parseFileName(extension=".tlt")) ts.generateTltFile(angleFilePath, excludeViews=True) - def applyNewStackBasic(self, ts, outputTsFileName, inputTsFileName, xfFile, doSwap): + def applyNewStackBasic(self, ts, outputTsFileName, inputTsFileName, xfFile, doSwap, tsExcludedIndices=None): argsAlignment, paramsAlignment = self.getBasicNewstackParams(ts, outputTsFileName, inputTsFileName=inputTsFileName, xfFile=xfFile, firstItem=ts.getFirstItem(), - doSwap=doSwap) + doSwap=doSwap, + tsExcludedIndices=tsExcludedIndices) Plugin.runImod(self, 'newstack', argsAlignment % paramsAlignment) @staticmethod def getBasicNewstackParams(ts, outputTsFileName, inputTsFileName=None, - xfFile=None, firstItem=None, binning=1, doSwap=False): + xfFile=None, firstItem=None, binning=1, doSwap=False, tsExcludedIndices=None): """ Returns basic newstack arguments :param ts: Title Series object @@ -214,7 +217,7 @@ def getBasicNewstackParams(ts, outputTsFileName, inputTsFileName=None, :param firstItem: Optional, otherwise it will be taken from ts :param binning: Default to 1. to apply to output size :param doSwap: Default False. - + :param tsExcludedIndices: List of indices to be excluded in the tilt-series """ if firstItem is None: @@ -249,6 +252,10 @@ def getBasicNewstackParams(ts, outputTsFileName, inputTsFileName=None, argsAlignment += "-size %(size)s " + if tsExcludedIndices: + paramsAlignment["exclude"] = ",".join(map(str, tsExcludedIndices)) + argsAlignment += "-exclude %(exclude)s " + return argsAlignment, paramsAlignment # --------------------------- OUTPUT functions ---------------------------- diff --git a/imod/protocols/protocol_ctfCorrection.py b/imod/protocols/protocol_ctfCorrection.py index b5d3ca78..5d3031ed 100644 --- a/imod/protocols/protocol_ctfCorrection.py +++ b/imod/protocols/protocol_ctfCorrection.py @@ -164,19 +164,13 @@ def _initialize(self): def convertInputStep(self, tsId, **kwargs): oddEvenFlag = self.applyToOddEven(self.inputSetOfTiltSeries.get()) # Considering swapXY is required to make tilt axis vertical - super().convertInputStep(tsId, doSwap=True, oddEven=oddEvenFlag, onlyEnabled=True) + super().convertInputStep(tsId, doSwap=True, oddEven=oddEvenFlag) def generateDefocusFile(self, tsId): ts = self.tsDict[tsId] ctfTomoSeries = self.ctfDict[tsId] self.debug(f"Generating defocus file for {tsId} (ObjId), {tsId} (TsId)") - # # At this step, a tlt file has been generated considering the excluded views in the current TS. If the no. - # # tilt images is different from the no. CTFTomos, the attribute _isEnabled of the elements that are not present - # # in the TS or are disabled tilt-images will be set to False and considered when generating the defocus IMOD - # # files - # if len(ts) != len(ctfTomoSeries): - # ctfTomoSeries.updateCtfTomoEnableFromTs(ts) # Compose the defocus file path defocusFilePath = self.getDefocusFileName(ts) """Generate defocus file""" diff --git a/imod/utils.py b/imod/utils.py index cc7011d4..72af0a18 100644 --- a/imod/utils.py +++ b/imod/utils.py @@ -824,11 +824,11 @@ def generateDefocusIMODFileFromObject(ctfTomoSeries, defocusFilePath, with open(defocusFilePath, 'w') as f: lines = ["1\t0\t0.0\t0.0\t0.0\t3\n"] - for index, ti in enumerate(tiltSeries): + ind = 1 + for ti in tiltSeries: ctfTomo = ctfTomoSeries.getCtfTomoFromTi(ti) if ctfTomo: tiltAngle = ti.getTiltAngle() - ind = index + 1 newLine = ("%d\t%d\t%.2f\t%.2f\t%.1f\t%.1f\t%.2f\n" % ( ind, ind, @@ -841,6 +841,7 @@ def generateDefocusIMODFileFromObject(ctfTomoSeries, defocusFilePath, ctfTomo.getDefocusAngle())) lines.append(newLine) + ind += 1 f.writelines(lines) From 8162b22c082e883bdd38506195836edc73078a33 Mon Sep 17 00:00:00 2001 From: JorMaister Date: Wed, 27 Mar 2024 12:13:36 +0100 Subject: [PATCH 07/49] Update the acquisition order in the CTFTomo objects --- imod/protocols/protocol_base.py | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/imod/protocols/protocol_base.py b/imod/protocols/protocol_base.py index 8148bb26..0950a858 100644 --- a/imod/protocols/protocol_base.py +++ b/imod/protocols/protocol_base.py @@ -490,57 +490,56 @@ def parseTSDefocusFile(self, inputTs, defocusFilePath, newCTFTomoSeries): f"Defocus file flag {defocusFileFlag} is not supported. Only supported formats " "correspond to flags 0, 1, 4, 5, and 37.") - excludedViews = inputTs.getExcludedViewsIndex() - ids = inputTs.getIdSet() - for index in ids: + for ti in inputTs: + tiObjId = ti.getObjId() newCTFTomo = CTFTomo() - newCTFTomo.setIndex(index) + newCTFTomo.setAcquisitionOrder(ti.getAcquisitonOrder()) - if index not in defocusUDict.keys() and index not in excludedViews: + if tiObjId not in defocusUDict.keys() and not ti.isEnabled(): raise IndexError("ERROR IN TILT-SERIES %s: NO CTF ESTIMATED FOR VIEW %d, TILT ANGLE %f" % ( - inputTs.getTsId(), index, inputTs[index].getTiltAngle())) + inputTs.getTsId(), tiObjId, inputTs[tiObjId].getTiltAngle())) " Plain estimation (any defocus flag)" newCTFTomo._defocusUList = CsvList(pType=float) - newCTFTomo.setDefocusUList(defocusUDict.get(index, [0.])) + newCTFTomo.setDefocusUList(defocusUDict.get(tiObjId, [0.])) if defocusFileFlag == 1: " Astigmatism estimation " newCTFTomo._defocusVList = CsvList(pType=float) - newCTFTomo.setDefocusVList(defocusVDict.get(index, [0.])) + newCTFTomo.setDefocusVList(defocusVDict.get(tiObjId, [0.])) newCTFTomo._defocusAngleList = CsvList(pType=float) - newCTFTomo.setDefocusAngleList(defocusAngleDict.get(index, [0.])) + newCTFTomo.setDefocusAngleList(defocusAngleDict.get(tiObjId, [0.])) elif defocusFileFlag == 4: " Phase-shift information " newCTFTomo._phaseShiftList = CsvList(pType=float) - newCTFTomo.setPhaseShiftList(phaseShiftDict.get(index, [0.])) + newCTFTomo.setPhaseShiftList(phaseShiftDict.get(tiObjId, [0.])) elif defocusFileFlag == 5: " Astigmatism and phase shift estimation " newCTFTomo._defocusVList = CsvList(pType=float) - newCTFTomo.setDefocusVList(defocusVDict.get(index, [0.])) + newCTFTomo.setDefocusVList(defocusVDict.get(tiObjId, [0.])) newCTFTomo._defocusAngleList = CsvList(pType=float) - newCTFTomo.setDefocusAngleList(defocusAngleDict.get(index, [0.])) + newCTFTomo.setDefocusAngleList(defocusAngleDict.get(tiObjId, [0.])) newCTFTomo._phaseShiftList = CsvList(pType=float) - newCTFTomo.setPhaseShiftList(phaseShiftDict.get(index, [0.])) + newCTFTomo.setPhaseShiftList(phaseShiftDict.get(tiObjId, [0.])) elif defocusFileFlag == 37: " Astigmatism, phase shift and cut-on frequency estimation " newCTFTomo._defocusVList = CsvList(pType=float) - newCTFTomo.setDefocusVList(defocusVDict.get(index, [0.])) + newCTFTomo.setDefocusVList(defocusVDict.get(tiObjId, [0.])) newCTFTomo._defocusAngleList = CsvList(pType=float) - newCTFTomo.setDefocusAngleList(defocusAngleDict.get(index, [0.])) + newCTFTomo.setDefocusAngleList(defocusAngleDict.get(tiObjId, [0.])) newCTFTomo._phaseShiftList = CsvList(pType=float) - newCTFTomo.setPhaseShiftList(phaseShiftDict.get(index, [0.])) + newCTFTomo.setPhaseShiftList(phaseShiftDict.get(tiObjId, [0.])) newCTFTomo._cutOnFreqList = CsvList(pType=float) - newCTFTomo.setCutOnFreqList(cutOnFreqDict.get(index, [0.])) + newCTFTomo.setCutOnFreqList(cutOnFreqDict.get(tiObjId, [0.])) newCTFTomo.completeInfoFromList() newCTFTomoSeries.append(newCTFTomo) From 49400f3471573f0d852edf1cbdc289f867ac1318 Mon Sep 17 00:00:00 2001 From: JorMaister Date: Wed, 27 Mar 2024 16:53:23 +0100 Subject: [PATCH 08/49] Update changes and reqs, bump version --- CHANGES.txt | 3 +++ imod/__init__.py | 2 +- requirements.txt | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 4aec7ef1..6fa2053f 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,3 +1,6 @@ +3.4.2: + Developers: + - Update the acquisition order in the CTFTomo objects (field added to that class in scipion-em-tomo v3.7.0). 3.3.0: - bugfix for import ctf, set missing defocus flag - move plugin-specific import CTF protocol to the core diff --git a/imod/__init__.py b/imod/__init__.py index 118f6e0e..5b67fb2d 100644 --- a/imod/__init__.py +++ b/imod/__init__.py @@ -35,7 +35,7 @@ from .constants import IMOD_HOME, ETOMO_CMD, DEFAULT_VERSION, VERSIONS, IMOD_VIEWER_BINNING -__version__ = '3.4.1' +__version__ = '3.4.2' _logo = "icon.png" _references = ['Kremer1996', 'Mastronarde2017'] diff --git a/requirements.txt b/requirements.txt index d0dcd66b..5ab7c8c3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -scipion-em-tomo \ No newline at end of file +scipion-em-tomo>=3.7.0 \ No newline at end of file From f3023625d954d7fbfd912be8aa3b02a964b3393f Mon Sep 17 00:00:00 2001 From: JorMaister Date: Wed, 27 Mar 2024 16:55:29 +0100 Subject: [PATCH 09/49] small fix --- imod/protocols/protocol_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imod/protocols/protocol_base.py b/imod/protocols/protocol_base.py index 0950a858..60c7ea25 100644 --- a/imod/protocols/protocol_base.py +++ b/imod/protocols/protocol_base.py @@ -493,7 +493,7 @@ def parseTSDefocusFile(self, inputTs, defocusFilePath, newCTFTomoSeries): for ti in inputTs: tiObjId = ti.getObjId() newCTFTomo = CTFTomo() - newCTFTomo.setAcquisitionOrder(ti.getAcquisitonOrder()) + newCTFTomo.setAcquisitionOrder(ti.getAcquisitionOrder()) if tiObjId not in defocusUDict.keys() and not ti.isEnabled(): raise IndexError("ERROR IN TILT-SERIES %s: NO CTF ESTIMATED FOR VIEW %d, TILT ANGLE %f" % ( From 1af13ba1354dd9448b4b7d0d4b0d7661954a346e Mon Sep 17 00:00:00 2001 From: JorMaister Date: Wed, 27 Mar 2024 17:02:17 +0100 Subject: [PATCH 10/49] Utils: fix the calls to the old bad defined getIndex method from CTFTomo --- imod/utils.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/imod/utils.py b/imod/utils.py index 0f764ec5..970df03b 100644 --- a/imod/utils.py +++ b/imod/utils.py @@ -827,7 +827,7 @@ def generateDefocusIMODFileFromObject(ctfTomoSeries, defocusFilePath, # upside down Scipion (highest index for # the tilt-image with the highest negative angle) for ctfTomo in ctfTomoSeries: - index = ctfTomo.getIndex().get() + index = ctfTomo.getIndex() newLine = ("%d\t%d\t%.2f\t%.2f\t%.1f\t%.1f\t%.2f\n" % ( index, @@ -857,7 +857,7 @@ def generateDefocusUDictionary(ctfTomoSeries): else ctfTomo.getDefocusVList() defocusInfoList = defocusInfoList.split(",") - index = ctfTomo.getIndex().get() + index = ctfTomo.getIndex() defocusUDict[index] = defocusInfoList @@ -875,7 +875,7 @@ def generateDefocusVDictionary(ctfTomoSeries): defocusInfoList = ctfTomo.getDefocusVList() defocusInfoList = defocusInfoList.split(",") - index = ctfTomo.getIndex().get() + index = ctfTomo.getIndex() defocusVDict[index] = defocusInfoList @@ -893,7 +893,7 @@ def generateDefocusAngleDictionary(ctfTomoSeries): defocusAngleList = ctfTomo.getDefocusAngleList() defocusAngleList = defocusAngleList.split(",") - index = ctfTomo.getIndex().get() + index = ctfTomo.getIndex() defocusAngleDict[index] = defocusAngleList @@ -911,7 +911,7 @@ def generatePhaseShiftDictionary(ctfTomoSeries): phaseShiftList = ctfTomo.getPhaseShiftList() phaseShiftList = phaseShiftList.split(",") - index = ctfTomo.getIndex().get() + index = ctfTomo.getIndex() phaseShiftDict[index] = phaseShiftList @@ -929,7 +929,7 @@ def generateCutOnFreqDictionary(ctfTomoSeries): cutOnFreqList = ctfTomo.getCutOnFreqList() cutOnFreqList = cutOnFreqList.split(",") - index = ctfTomo.getIndex().get() + index = ctfTomo.getIndex() cutOnFreqDict[index] = cutOnFreqList From 85c7e583e7abb51faf4c4bf1d62efe94f0ef9e84 Mon Sep 17 00:00:00 2001 From: JorMaister Date: Wed, 27 Mar 2024 17:12:34 +0100 Subject: [PATCH 11/49] small fix, tests passing --- imod/protocols/protocol_base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/imod/protocols/protocol_base.py b/imod/protocols/protocol_base.py index 60c7ea25..b5e4b922 100644 --- a/imod/protocols/protocol_base.py +++ b/imod/protocols/protocol_base.py @@ -494,6 +494,7 @@ def parseTSDefocusFile(self, inputTs, defocusFilePath, newCTFTomoSeries): tiObjId = ti.getObjId() newCTFTomo = CTFTomo() newCTFTomo.setAcquisitionOrder(ti.getAcquisitionOrder()) + newCTFTomo.setIndex(ti.getIndex()) if tiObjId not in defocusUDict.keys() and not ti.isEnabled(): raise IndexError("ERROR IN TILT-SERIES %s: NO CTF ESTIMATED FOR VIEW %d, TILT ANGLE %f" % ( From d6d889b59554c9a98b2fd49fed36370fa61bc2b1 Mon Sep 17 00:00:00 2001 From: fede-pe Date: Mon, 1 Apr 2024 15:47:40 +0200 Subject: [PATCH 12/49] save non residual landmarks with Nan --- imod/protocols/protocol_fiducialAlignment.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/imod/protocols/protocol_fiducialAlignment.py b/imod/protocols/protocol_fiducialAlignment.py index f261fbc0..38c82320 100644 --- a/imod/protocols/protocol_fiducialAlignment.py +++ b/imod/protocols/protocol_fiducialAlignment.py @@ -711,8 +711,8 @@ def computeOutputModelsStep(self, tsObjId): yCoor=fiducial[1], tiltIm=fiducial[2] + 1, chainId=chainId, - xResid='0', - yResid='0') + xResid=float('nan'), + yResid=float('nan')) output.append(landmarkModelNoGaps) output.update(landmarkModelNoGaps) From b1f1d66e41625405ca0e202adc6745b3cc9fb91f Mon Sep 17 00:00:00 2001 From: JorMaister Date: Mon, 1 Apr 2024 18:59:32 +0200 Subject: [PATCH 13/49] Add a method to intersect the present elements in the TS and the CTF and versions of existing methods adapted to consume them --- imod/protocols/protocol_base.py | 15 +++--- imod/protocols/protocol_ctfCorrection.py | 4 +- imod/utils.py | 59 ++++++++++++++++++------ 3 files changed, 56 insertions(+), 22 deletions(-) diff --git a/imod/protocols/protocol_base.py b/imod/protocols/protocol_base.py index e1f9684e..b308eb32 100644 --- a/imod/protocols/protocol_base.py +++ b/imod/protocols/protocol_base.py @@ -113,7 +113,8 @@ def getTmpTSFile(self, tsId, tmpPrefix=None, suffix=".mrcs"): return os.path.join(tmpPrefix, tsId + suffix) - def convertInputStep(self, tsObjId, generateAngleFile=True, imodInterpolation=True, doSwap=False, oddEven=False): + def convertInputStep(self, tsObjId, generateAngleFile=True, imodInterpolation=True, doSwap=False, + oddEven=False, presentAcqOrders=None): """ :param tsObjId: Tilt series identifier :param generateAngleFile: Boolean(True) to generate IMOD angle file @@ -122,6 +123,8 @@ def convertInputStep(self, tsObjId, generateAngleFile=True, imodInterpolation=Tr Pass None to cancel interpolation. :param doSwap: if applying alignment, consider swapping X/Y :param oddEven: process odd/even sets + :param presentAcqOrders: set containing the present acq orders in both the given TS and CTFTomoSeries. Used + to generate the xf file, the tlt file, and the interpolated TS with IMOD's newstack program. """ if type(tsObjId) is str: ts = self.tsDict[tsObjId] @@ -132,8 +135,7 @@ def convertInputStep(self, tsObjId, generateAngleFile=True, imodInterpolation=Tr tsId = ts.getTsId() extraPrefix = self._getExtraPath(tsId) tmpPrefix = self._getTmpPath(tsId) - path.makePath(tmpPrefix) - path.makePath(extraPrefix) + path.makePath(*[tmpPrefix, extraPrefix]) firstTi = ts.getFirstItem() inTsFileName = firstTi.getFileName() @@ -161,11 +163,11 @@ def convertInputStep(self, tsObjId, generateAngleFile=True, imodInterpolation=Tr if firstTi.hasTransform(): # Generate transformation matrices file (xf) xfFile = os.path.join(extraPrefix, firstTi.parseFileName(extension=".xf")) - utils.formatTransformFile(ts, xfFile) + utils.genXfFile(ts, xfFile) # Generate the interpolated TS with IMOD's newstack program logger.info("Tilt-series interpolated with IMOD [%s]" % tsId) - tsExcludedIndices = [ti.getIndex() for ti in ts if not ti.isEnabled()] + tsExcludedIndices = [ti.getIndex() for ti in ts if not ti.getAcquisitionOrder() in presentAcqOrders] self.applyNewStackBasic(ts, outputTsFileName, inTsFileName, xfFile, doSwap, tsExcludedIndices=tsExcludedIndices) if oddEven: @@ -175,6 +177,7 @@ def convertInputStep(self, tsObjId, generateAngleFile=True, imodInterpolation=Tr tsExcludedIndices=tsExcludedIndices) else: + # The given TS is interpolated logger.info("Tilt-series linked [%s]" % tsId) path.createLink(firstTi.getFileName(), outputTsFileName) @@ -193,7 +196,7 @@ def convertInputStep(self, tsObjId, generateAngleFile=True, imodInterpolation=Tr if generateAngleFile: logger.info("Generate angle file for the tilt-series [%s]" % tsId) angleFilePath = os.path.join(extraPrefix, firstTi.parseFileName(extension=".tlt")) - ts.generateTltFile(angleFilePath, excludeViews=True) + ts.genTltFile(angleFilePath, presentAcqOrders=presentAcqOrders) def applyNewStackBasic(self, ts, outputTsFileName, inputTsFileName, xfFile, doSwap, tsExcludedIndices=None): argsAlignment, paramsAlignment = self.getBasicNewstackParams(ts, diff --git a/imod/protocols/protocol_ctfCorrection.py b/imod/protocols/protocol_ctfCorrection.py index 5d3031ed..5609a86a 100644 --- a/imod/protocols/protocol_ctfCorrection.py +++ b/imod/protocols/protocol_ctfCorrection.py @@ -31,6 +31,7 @@ import pyworkflow.utils.path as path from pwem.emlib.image import ImageHandler from tomo.objects import TiltSeries, TiltImage +from tomo.utils import getCommonTsAndCtfElements from .. import Plugin, utils from .protocol_base import ProtImodBase, EXT_MRCS_TS_ODD_NAME, EXT_MRCS_TS_EVEN_NAME @@ -162,9 +163,10 @@ def _initialize(self): self.ctfDict = {ctf.getTsId(): ctf.clone(ignoreAttrs=[]) for ctf in self.inputSetOfCtfTomoSeries.get()} def convertInputStep(self, tsId, **kwargs): + presentAcqOrders = getCommonTsAndCtfElements(self.tsDict[tsId], self.ctfDict[tsId]) oddEvenFlag = self.applyToOddEven(self.inputSetOfTiltSeries.get()) # Considering swapXY is required to make tilt axis vertical - super().convertInputStep(tsId, doSwap=True, oddEven=oddEvenFlag) + super().convertInputStep(tsId, doSwap=True, oddEven=oddEvenFlag, presentAcqOrders=presentAcqOrders) def generateDefocusFile(self, tsId): ts = self.tsDict[tsId] diff --git a/imod/utils.py b/imod/utils.py index 72af0a18..f160617b 100644 --- a/imod/utils.py +++ b/imod/utils.py @@ -30,11 +30,13 @@ import os import csv import math +from typing import Set, Union + import numpy as np import pyworkflow.object as pwobj import pyworkflow.utils as pwutils from imod import Plugin - +from tomo.objects import TiltSeries logger = logging.getLogger(__name__) @@ -63,6 +65,33 @@ def formatTransformFile(ts, transformFilePath, onlyEnabled=False): csvW.writerows(tsMatrixTransformList) +def genXfFile(ts: TiltSeries, outXfName: str, presentAcqOrders: Union[set, None] = None) -> None: + """ This method takes a tilt series and the output transformation file path + and creates an IMOD-based transform file in the location indicated. The transformation matrix + of a tilt-image is only added if its acquisition order is contained in a set composed of the + acquisition orders present in both the given tilt-series and the CTFTomoSeries. + """ + + def formatMatrix(tiltImage): + transform = tiltImage.getTransform().getMatrix().flatten() + transformIMOD = ['%.7f' % transform[0], + '%.7f' % transform[1], + '%.7f' % transform[3], + '%.7f' % transform[4], + "{:>6}".format('%.3g' % transform[2]), + "{:>6}".format('%.3g' % transform[5])] + return transformIMOD + + if presentAcqOrders: + tsMatrixList = [formatMatrix(ti) for ti in ts if ti.getAcquisitionOrder() in presentAcqOrders] + else: + tsMatrixList = [formatMatrix(ti) for ti in ts] + + with open(outXfName, 'w') as f: + csvW = csv.writer(f, delimiter='\t') + csvW.writerows(tsMatrixList) + + def formatTransformFileFromTransformList(transformMatrixList, transformFilePath): """ This method takes a list of Transform matrices and the output transformation file path and creates an @@ -167,7 +196,7 @@ def generateIMODFiducialTextFile(landmarkModel, outputFilePath): for vector in infoTable: outputLines.append("\t%s\t%s\t%s\t%s\n" % (vector[3], vector[0], - vector[1], int(vector[2])-1)) + vector[1], int(vector[2]) - 1)) with open(outputFilePath, 'w') as f: f.writelines(outputLines) @@ -649,7 +678,7 @@ def generateDefocusIMODFileFromObject(ctfTomoSeries, defocusFilePath, round(tiltSeries[index + ctfTomoSeries.getNumberOfEstimationsInRange()].getTiltAngle(), 2), round(tiltSeries[index].getTiltAngle(), 2), # CONVERT DEFOCUS VALUE TO NANOMETERS (IMOD CONVENTION) - int(float(defocusUDict[index][0])/10) + int(float(defocusUDict[index][0]) / 10) )) lines.append(newLine) @@ -688,9 +717,9 @@ def generateDefocusIMODFileFromObject(ctfTomoSeries, defocusFilePath, round(tiltSeries[index + ctfTomoSeries.getNumberOfEstimationsInRange()].getTiltAngle(), 2), round(tiltSeries[index].getTiltAngle(), 2), # CONVERT DEFOCUS VALUE TO NANOMETERS (IMOD CONVENTION) - float(defocusUDict[index][0])/10, + float(defocusUDict[index][0]) / 10, # CONVERT DEFOCUS VALUE TO NANOMETERS (IMOD CONVENTION) - float(defocusVDict[index][0])/10, + float(defocusVDict[index][0]) / 10, float(defocusAngleDict[index][0]), )) @@ -724,7 +753,7 @@ def generateDefocusIMODFileFromObject(ctfTomoSeries, defocusFilePath, round(tiltSeries[index + ctfTomoSeries.getNumberOfEstimationsInRange()].getTiltAngle(), 2), round(tiltSeries[index].getTiltAngle(), 2), # CONVERT DEFOCUS VALUE TO NANOMETERS (IMOD CONVENTION) - float(defocusUDict[index][0])/10, + float(defocusUDict[index][0]) / 10, float(phaseShiftDict[index][0]), )) @@ -745,7 +774,7 @@ def generateDefocusIMODFileFromObject(ctfTomoSeries, defocusFilePath, with open(defocusFilePath, 'w') as f: # This line is added at the beginning of the file in order # to match the IMOD defocus file format - lines = ["5\t0\t0.0\t0.0\t0.0\t3\n"] + lines = ["5\t0\t0.0\t0.0\t0.0\t3\n"] for index in defocusUDict.keys(): @@ -761,9 +790,9 @@ def generateDefocusIMODFileFromObject(ctfTomoSeries, defocusFilePath, round(tiltSeries[index + ctfTomoSeries.getNumberOfEstimationsInRange()].getTiltAngle(), 2), round(tiltSeries[index].getTiltAngle(), 2), # CONVERT DEFOCUS VALUE TO NANOMETERS (IMOD CONVENTION) - float(defocusUDict[index][0])/10, + float(defocusUDict[index][0]) / 10, # CONVERT DEFOCUS VALUE TO NANOMETERS (IMOD CONVENTION) - float(defocusVDict[index][0])/10, + float(defocusVDict[index][0]) / 10, float(defocusAngleDict[index][0]), float(phaseShiftDict[index][0]) )) @@ -801,9 +830,9 @@ def generateDefocusIMODFileFromObject(ctfTomoSeries, defocusFilePath, round(tiltSeries[index + ctfTomoSeries.getNumberOfEstimationsInRange()].getTiltAngle(), 2), round(tiltSeries[index].getTiltAngle(), 2), # CONVERT DEFOCUS VALUE TO NANOMETERS (IMOD CONVENTION) - float(defocusUDict[index][0])/10, + float(defocusUDict[index][0]) / 10, # CONVERT DEFOCUS VALUE TO NANOMETERS (IMOD CONVENTION) - float(defocusVDict[index][0])/10, + float(defocusVDict[index][0]) / 10, float(defocusAngleDict[index][0]), float(phaseShiftDict[index][0]), float(cutOnFreqDict[index][0]) @@ -835,9 +864,9 @@ def generateDefocusIMODFileFromObject(ctfTomoSeries, defocusFilePath, tiltAngle, tiltAngle, # CONVERT DEFOCUS VALUE TO NANOMETERS (IMOD CONVENTION) - ctfTomo.getDefocusU()/10, + ctfTomo.getDefocusU() / 10, # CONVERT DEFOCUS VALUE TO NANOMETERS (IMOD CONVENTION) - ctfTomo.getDefocusV()/10, + ctfTomo.getDefocusV() / 10, ctfTomo.getDefocusAngle())) lines.append(newLine) @@ -966,7 +995,7 @@ def calculateRotationAngleFromTM(ts): tm = ti.getTransform().getMatrix() cosRotationAngle = tm[0][0] sinRotationAngle = tm[1][0] - avgRotationAngle += math.degrees(math.atan(sinRotationAngle/cosRotationAngle)) + avgRotationAngle += math.degrees(math.atan(sinRotationAngle / cosRotationAngle)) avgRotationAngle = avgRotationAngle / ts.getSize() @@ -984,7 +1013,7 @@ def generateDoseFileFromDoseTS(ts, doseFileOutputPath): for ti in ts: acq = ti.getAcquisition() - doseInfoList.append((acq.getAccumDose()-acq.getDosePerFrame(), acq.getDosePerFrame())) + doseInfoList.append((acq.getAccumDose() - acq.getDosePerFrame(), acq.getDosePerFrame())) np.savetxt(doseFileOutputPath, np.asarray(doseInfoList), fmt='%f', delimiter=" ") From f6df2f556be49dd3422f16531305fdfff86355ee Mon Sep 17 00:00:00 2001 From: JorMaister Date: Tue, 2 Apr 2024 18:45:38 +0200 Subject: [PATCH 14/49] CTFCorrection, all the cases working excepting the TS unmodified and the CTF re-stacked --- imod/protocols/protocol_base.py | 95 ++++++++++++----- imod/protocols/protocol_ctfCorrection.py | 129 ++++++++++++----------- imod/utils.py | 4 +- 3 files changed, 136 insertions(+), 92 deletions(-) diff --git a/imod/protocols/protocol_base.py b/imod/protocols/protocol_base.py index b308eb32..c55e76fc 100644 --- a/imod/protocols/protocol_base.py +++ b/imod/protocols/protocol_base.py @@ -36,10 +36,8 @@ CTFTomo, SetOfTiltSeriesCoordinates, TiltImage) from .. import Plugin, utils - logger = logging.getLogger(__name__) - OUTPUT_TS_COORDINATES_NAME = "TiltSeriesCoordinates" OUTPUT_FIDUCIAL_NO_GAPS_NAME = "FiducialModelNoGaps" OUTPUT_FIDUCIAL_GAPS_NAME = "FiducialModelGaps" @@ -49,10 +47,16 @@ OUTPUT_CTF_SERIE = "CTFTomoSeries" OUTPUT_TOMOGRAMS_NAME = "Tomograms" OUTPUT_COORDINATES_3D_NAME = "Coordinates3D" -EXT_MRCS_TS_EVEN_NAME = "_even.mrcs" -EXT_MRCS_TS_ODD_NAME = "_odd.mrcs" -EXT_MRC_EVEN_NAME = "_even.mrc" -EXT_MRC_ODD_NAME = "_odd.mrc" +EXT_MRCS_TS_EVEN_NAME = "even.mrcs" +EXT_MRCS_TS_ODD_NAME = "odd.mrcs" +EXT_MRC_EVEN_NAME = "even.mrc" +EXT_MRC_ODD_NAME = "odd.mrc" +EVEN = 'even' +ODD = 'odd' +MRCS_EXT = 'mrcs' +XF_EXT = 'xf' +TLT_EXT = 'tlt' +DEFOCUS_EXT = 'defocus' class ProtImodBase(ProtTomoImportFiles, EMProtocol, ProtTomoBase): @@ -113,10 +117,24 @@ def getTmpTSFile(self, tsId, tmpPrefix=None, suffix=".mrcs"): return os.path.join(tmpPrefix, tsId + suffix) + def genTsPaths(self, tsId): + """Generate the subdirectories corresponding to the current tilt-series in tmp and extra""" + path.makePath(*[self._getExtraPath(tsId), self._getTmpPath(tsId)]) + + @staticmethod + def getOutTsFileName(tsId, suffix=None, ext='mrc'): + return f'{tsId}_{suffix}.{ext}' if suffix else f'{tsId}.{ext}' + + def getTmpOutFile(self, tsId, suffix=None, ext='mrc'): + return self._getTmpPath(tsId, self.getOutTsFileName(tsId, suffix=suffix, ext=ext)) + + def getExtraOutFile(self, tsId, suffix=None, ext='mrc'): + return self._getExtraPath(tsId, self.getOutTsFileName(tsId, suffix=suffix, ext=ext)) + def convertInputStep(self, tsObjId, generateAngleFile=True, imodInterpolation=True, doSwap=False, oddEven=False, presentAcqOrders=None): """ - :param tsObjId: Tilt series identifier + :param tsObjId: Tilt-series identifier :param generateAngleFile: Boolean(True) to generate IMOD angle file :param imodInterpolation: Boolean (True) to interpolate the tilt series with imod in case there is a TM. @@ -132,37 +150,50 @@ def convertInputStep(self, tsObjId, generateAngleFile=True, imodInterpolation=Tr tsSet = self.inputSetOfTiltSeries, ts = tsSet.get()[tsObjId] if isinstance(tsSet, Pointer) else tsSet[tsObjId] - tsId = ts.getTsId() - extraPrefix = self._getExtraPath(tsId) - tmpPrefix = self._getTmpPath(tsId) - path.makePath(*[tmpPrefix, extraPrefix]) + self.genTsPaths(ts.getTsId()) + self.genAlignmentFiles(ts, generateAngleFile=generateAngleFile, imodInterpolation=imodInterpolation, + doSwap=doSwap, oddEven=oddEven, presentAcqOrders=presentAcqOrders) + def genAlignmentFiles(self, ts, generateAngleFile=True, imodInterpolation=True, doSwap=False, + oddEven=False, presentAcqOrders=None): + """ + :param ts: Tilt-series + :param generateAngleFile: Boolean(True) to generate IMOD angle file + :param imodInterpolation: Boolean (True) to interpolate the tilt series with + imod in case there is a TM. + Pass None to cancel interpolation. + :param doSwap: if applying alignment, consider swapping X/Y + :param oddEven: process odd/even sets + :param presentAcqOrders: set containing the present acq orders in both the given TS and CTFTomoSeries. Used + to generate the xf file, the tlt file, and the interpolated TS with IMOD's newstack program. + """ + # Initialization + tsId = ts.getTsId() firstTi = ts.getFirstItem() inTsFileName = firstTi.getFileName() - outputTsFileName = os.path.join(tmpPrefix, firstTi.parseFileName()) + outputTsFileName = self.getTmpOutFile(tsId) + fnOdd = None + fnEven = None + outputOddTsFileName = None + outputEvenTsFileName = None + if oddEven: + fnOdd = ts.getOddFileName() + fnEven = ts.getEvenFileName() + outputOddTsFileName = self.getTmpOutFile(tsId, suffix=ODD, ext=MRCS_EXT) + outputEvenTsFileName = self.getTmpOutFile(tsId, suffix=EVEN, ext=MRCS_EXT) # Interpolation if imodInterpolation is None: logger.info("Tilt series %s linked." % tsId) path.createLink(inTsFileName, outputTsFileName) + elif imodInterpolation: logger.info("Apply the transformation form the input tilt-series") - # Odd / Even - outputOddTsFileName = None - outputEvenTsFileName = None - fnOdd = None - fnEven = None - if oddEven: - fnOdd = ts.getOddFileName() - fnEven = ts.getEvenFileName() - outputOddTsFileName = self.getTmpTSFile(tsId, tmpPrefix=tmpPrefix, suffix=EXT_MRCS_TS_ODD_NAME) - outputEvenTsFileName = self.getTmpTSFile(tsId, tmpPrefix=tmpPrefix, suffix=EXT_MRCS_TS_EVEN_NAME) - # Use IMOD newstack interpolation if firstTi.hasTransform(): # Generate transformation matrices file (xf) - xfFile = os.path.join(extraPrefix, firstTi.parseFileName(extension=".xf")) + xfFile = self.getExtraOutFile(tsId, ext=XF_EXT) utils.genXfFile(ts, xfFile) # Generate the interpolated TS with IMOD's newstack program @@ -176,6 +207,11 @@ def convertInputStep(self, tsObjId, generateAngleFile=True, imodInterpolation=Tr self.applyNewStackBasic(ts, outputEvenTsFileName, fnEven, xfFile, doSwap, tsExcludedIndices=tsExcludedIndices) + # If some views were excluded to generate the new stack, a new xfFile containing them should be + # generated + if len(ts) != len(presentAcqOrders): + utils.genXfFile(ts, xfFile, presentAcqOrders=presentAcqOrders) + else: # The given TS is interpolated logger.info("Tilt-series linked [%s]" % tsId) @@ -195,7 +231,7 @@ def convertInputStep(self, tsObjId, generateAngleFile=True, imodInterpolation=Tr # Generate the tlt file if generateAngleFile: logger.info("Generate angle file for the tilt-series [%s]" % tsId) - angleFilePath = os.path.join(extraPrefix, firstTi.parseFileName(extension=".tlt")) + angleFilePath = self.getExtraOutFile(tsId, ext=TLT_EXT) ts.genTltFile(angleFilePath, presentAcqOrders=presentAcqOrders) def applyNewStackBasic(self, ts, outputTsFileName, inputTsFileName, xfFile, doSwap, tsExcludedIndices=None): @@ -258,6 +294,8 @@ def getBasicNewstackParams(ts, outputTsFileName, inputTsFileName=None, if tsExcludedIndices: paramsAlignment["exclude"] = ",".join(map(str, tsExcludedIndices)) argsAlignment += "-exclude %(exclude)s " + # From IMOD's newstack doc: "sections are numbered from 0 unless -fromone is entered" + argsAlignment += "-fromone " return argsAlignment, paramsAlignment @@ -471,7 +509,8 @@ def getOutputSetOfCTFTomoSeries(self, outputSetName): return outputSetOfCTFTomoSeries - def parseTSDefocusFile(self, inputTs, defocusFilePath, newCTFTomoSeries): + @staticmethod + def parseTSDefocusFile(inputTs, defocusFilePath, newCTFTomoSeries): """ Parse tilt-series ctf estimation file. :param inputTs: input tilt-series :param defocusFilePath: input *.defocus file to be parsed @@ -571,7 +610,7 @@ def parseTSDefocusFile(self, inputTs, defocusFilePath, newCTFTomoSeries): newCTFTomoSeries.calculateDefocusUDeviation(defocusUTolerance=20) newCTFTomoSeries.calculateDefocusVDeviation(defocusVTolerance=20) - def createOutputFailedSet(self, ts): + def createOutputFailedSet(self, ts, presentAcqOrders=None): # Check if the tilt-series ID is in the failed tilt-series # list to add it to the set tsId = ts.getTsId() @@ -583,6 +622,8 @@ def createOutputFailedSet(self, ts): output.append(newTs) for tiltImage in ts: + if presentAcqOrders and tiltImage.getAcquisitionOrder() not in presentAcqOrders: + continue newTi = TiltImage() newTi.copyInfo(tiltImage, copyId=True, copyTM=True) newTi.setAcquisition(tiltImage.getAcquisition()) diff --git a/imod/protocols/protocol_ctfCorrection.py b/imod/protocols/protocol_ctfCorrection.py index 5609a86a..fd4375d5 100644 --- a/imod/protocols/protocol_ctfCorrection.py +++ b/imod/protocols/protocol_ctfCorrection.py @@ -24,16 +24,14 @@ # * e-mail address 'scipion@cnb.csic.es' # * # ***************************************************************************** -from os.path import join from pyworkflow import BETA from pyworkflow.object import Set, String import pyworkflow.protocol.params as params -import pyworkflow.utils.path as path from pwem.emlib.image import ImageHandler from tomo.objects import TiltSeries, TiltImage from tomo.utils import getCommonTsAndCtfElements from .. import Plugin, utils -from .protocol_base import ProtImodBase, EXT_MRCS_TS_ODD_NAME, EXT_MRCS_TS_EVEN_NAME +from .protocol_base import ProtImodBase, DEFOCUS_EXT, TLT_EXT, XF_EXT, ODD, MRCS_EXT, EVEN class ProtImodCtfCorrection(ProtImodBase): @@ -139,16 +137,28 @@ def _defineParams(self, form): # -------------------------- INSERT steps functions ----------------------- def _insertAllSteps(self): + # JORGE + import os + fname = "/home/jjimenez/test_JJ.txt" + if os.path.exists(fname): + os.remove(fname) + fjj = open(fname, "a+") + fjj.write('JORGE--------->onDebugMode PID {}'.format(os.getpid())) + fjj.close() + print('JORGE--------->onDebugMode PID {}'.format(os.getpid())) + import time + time.sleep(10) + # JORGE_END nonMatchingTsIds = [] self._initialize() for tsId in self.tsDict.keys(): # Stores the steps serializing the tsId instead of the whole ts object matchingCtfTomoSeries = self.ctfDict.get(tsId, None) if matchingCtfTomoSeries: - self._insertFunctionStep(self.convertInputStep, tsId) - self._insertFunctionStep(self.generateDefocusFile, tsId) + presentAcqOrders = getCommonTsAndCtfElements(self.tsDict[tsId], self.ctfDict[tsId]) + self._insertFunctionStep(self.convertInputsStep, tsId, presentAcqOrders) self._insertFunctionStep(self.ctfCorrection, tsId) - self._insertFunctionStep(self.createOutputStep, tsId) - self._insertFunctionStep(self.createOutputFailedSet, tsId) + self._insertFunctionStep(self.createOutputStep, tsId, presentAcqOrders) + self._insertFunctionStep(self.createOutputFailedStep, tsId, presentAcqOrders) else: nonMatchingTsIds.append(tsId) self._insertFunctionStep(self.closeOutputSetsStep) @@ -162,40 +172,30 @@ def _initialize(self): self.tsDict = {ts.getTsId(): ts.clone(ignoreAttrs=[]) for ts in self.inputSetOfTiltSeries.get()} self.ctfDict = {ctf.getTsId(): ctf.clone(ignoreAttrs=[]) for ctf in self.inputSetOfCtfTomoSeries.get()} - def convertInputStep(self, tsId, **kwargs): - presentAcqOrders = getCommonTsAndCtfElements(self.tsDict[tsId], self.ctfDict[tsId]) - oddEvenFlag = self.applyToOddEven(self.inputSetOfTiltSeries.get()) - # Considering swapXY is required to make tilt axis vertical - super().convertInputStep(tsId, doSwap=True, oddEven=oddEvenFlag, presentAcqOrders=presentAcqOrders) - - def generateDefocusFile(self, tsId): - ts = self.tsDict[tsId] - ctfTomoSeries = self.ctfDict[tsId] - - self.debug(f"Generating defocus file for {tsId} (ObjId), {tsId} (TsId)") - # Compose the defocus file path - defocusFilePath = self.getDefocusFileName(ts) - """Generate defocus file""" - utils.generateDefocusIMODFileFromObject(ctfTomoSeries, defocusFilePath, inputTiltSeries=ts) + def convertInputsStep(self, tsId, presentAcqOrders): + # Generate the alignment-related files: xf, tlt, and a possible mrc + super().convertInputStep(tsId, # Considering swapXY is required to make tilt axis vertical + doSwap=True, + oddEven=self.applyToOddEven(self.inputSetOfTiltSeries.get()), + presentAcqOrders=presentAcqOrders) + # Generate the defocus file + self.generateDefocusFile(tsId, presentAcqOrders=presentAcqOrders) @ProtImodBase.tryExceptDecorator def ctfCorrection(self, tsId): ts = self.tsDict[tsId] acq = ts.getAcquisition() - extraPrefix = self._getExtraPath(tsId) - tmpPrefix = self._getTmpPath(tsId) - firstItem = ts.getFirstItem() """Run ctfphaseflip IMOD program""" paramsCtfPhaseFlip = { - 'inputStack': join(tmpPrefix, firstItem.parseFileName()), - 'angleFile': join(extraPrefix, firstItem.parseFileName(extension=".tlt")), - 'outputFileName': join(extraPrefix, firstItem.parseFileName()), - 'defocusFile': self.getDefocusFileName(ts), + 'inputStack': self.getTmpOutFile(tsId), + 'angleFile': self.getExtraOutFile(tsId, ext=TLT_EXT), + 'outputFileName': self.getExtraOutFile(tsId), + 'defocusFile': self.getExtraOutFile(tsId, ext=DEFOCUS_EXT), 'voltage': acq.getVoltage(), - 'sphericalAberration': ts.getAcquisition().getSphericalAberration(), + 'sphericalAberration': acq.getSphericalAberration(), 'defocusTol': self.defocusTol.get(), - 'pixelSize': ts.getSamplingRate() / 10, + 'pixelSize': ts.getSamplingRate() / 10, # nm/px 'amplitudeContrast': acq.getAmplitudeContrast(), 'interpolationWidth': self.interpolationWidth.get() } @@ -216,25 +216,23 @@ def ctfCorrection(self, tsId): "-ActionIfGPUFails 2,2 " if ts.getFirstItem().hasTransform(): - paramsCtfPhaseFlip['xformFile'] = join(extraPrefix, firstItem.parseFileName(extension=".xf")) + paramsCtfPhaseFlip['xformFile'] = self.getExtraOutFile(tsId, ext=XF_EXT) argsCtfPhaseFlip += "-TransformFile %(xformFile)s " Plugin.runImod(self, 'ctfphaseflip', argsCtfPhaseFlip % paramsCtfPhaseFlip) if self.applyToOddEven(ts): # ODD - oddFn = self.getTmpTSFile(tsId, tmpPrefix=tmpPrefix, suffix=EXT_MRCS_TS_ODD_NAME) - paramsCtfPhaseFlip['inputStack'] = oddFn - paramsCtfPhaseFlip['outputFileName'] = join(extraPrefix, tsId+EXT_MRCS_TS_ODD_NAME) + paramsCtfPhaseFlip['inputStack'] = self.getTmpOutFile(tsId, suffix=ODD, ext=MRCS_EXT) + paramsCtfPhaseFlip['outputFileName'] = self.getExtraOutFile(tsId, suffix=ODD, ext=MRCS_EXT) Plugin.runImod(self, 'ctfphaseflip', argsCtfPhaseFlip % paramsCtfPhaseFlip) # EVEN - evenFn = self.getTmpTSFile(tsId, tmpPrefix=tmpPrefix, suffix=EXT_MRCS_TS_EVEN_NAME) - paramsCtfPhaseFlip['inputStack'] = evenFn - paramsCtfPhaseFlip['outputFileName'] = join(extraPrefix, tsId+EXT_MRCS_TS_EVEN_NAME) + paramsCtfPhaseFlip['inputStack'] = self.getTmpOutFile(tsId, suffix=EVEN, ext=MRCS_EXT) + paramsCtfPhaseFlip['outputFileName'] = self.getExtraOutFile(tsId, suffix=EVEN, ext=MRCS_EXT) Plugin.runImod(self, 'ctfphaseflip', argsCtfPhaseFlip % paramsCtfPhaseFlip) - def createOutputStep(self, tsId): + def createOutputStep(self, tsId, presentAcqOrders): if tsId not in self._failedTs: inTsSet = self.inputSetOfTiltSeries.get() outputSetOfTs = self.getOutputSetOfTiltSeries(inTsSet) @@ -252,29 +250,30 @@ def createOutputStep(self, tsId): ih = ImageHandler() - for index, tiltImage in enumerate(ts): - newTi = TiltImage() - newTi.copyInfo(tiltImage, copyId=True, copyTM=False) - acq = tiltImage.getAcquisition() - acq.setTiltAxisAngle(0.) # Is interpolated - newTi.setAcquisition(acq) - newTi.setLocation(index + 1, (join(extraPrefix, tiltImage.parseFileName()))) - if self.applyToOddEven(ts): - locationOdd = index + 1, (join(extraPrefix, tsId + EXT_MRCS_TS_ODD_NAME)) - locationEven = index + 1, (join(extraPrefix, tsId + EXT_MRCS_TS_EVEN_NAME)) - newTi.setOddEven([ih.locationToXmipp(locationOdd), ih.locationToXmipp(locationEven)]) - else: - newTi.setOddEven([]) - newTs.append(newTi) + for index, inTi in enumerate(ts): + if inTi.getAcquisitionOrder() in presentAcqOrders: + newTi = TiltImage() + newTi.copyInfo(inTi, copyId=True, copyTM=False) + acq = inTi.getAcquisition() + acq.setTiltAxisAngle(0.) # Is interpolated + newTi.setAcquisition(acq) + newTi.setLocation(index + 1, self.getExtraOutFile(tsId)) + if self.applyToOddEven(ts): + locationOdd = index + 1, self.getExtraOutFile(tsId, suffix=ODD, ext=MRCS_EXT) + locationEven = index + 1, self.getExtraOutFile(tsId, suffix=EVEN, ext=MRCS_EXT) + newTi.setOddEven([ih.locationToXmipp(locationOdd), ih.locationToXmipp(locationEven)]) + else: + newTi.setOddEven([]) + newTs.append(newTi) newTs.write(properties=False) outputSetOfTs.update(newTs) outputSetOfTs.write() self._store(outputSetOfTs) - def createOutputFailedSet(self, tsId): + def createOutputFailedStep(self, tsId, presentAcqOrders): ts = self.tsDict[tsId] - super().createOutputFailedSet(ts) + super().createOutputFailedSet(ts, presentAcqOrders=presentAcqOrders) def closeOutputSetsStep(self): for _, output in self.iterOutputAttributes(): @@ -283,15 +282,17 @@ def closeOutputSetsStep(self): self._store() # --------------------------- UTILS functions ----------------------------- - def getDefocusFileName(self, ts): - """ Returns the path of the defocus filename based on - the tilt series and creates the folder/s""" - - extraPrefix = self._getExtraPath(ts.getTsId()) - path.makePath(extraPrefix) - defocusFn = ts.getFirstItem().parseFileName(extension=".defocus") - defocusFilePath = join(extraPrefix, defocusFn) - return defocusFilePath + def generateDefocusFile(self, tsId, presentAcqOrders=None): + ts = self.tsDict[tsId] + ctfTomoSeries = self.ctfDict[tsId] + + self.debug(f"Generating defocus file for {tsId} (ObjId), {tsId} (TsId)") + # Compose the defocus file path + defocusFilePath = self.getExtraOutFile(tsId, ext=DEFOCUS_EXT) + """Generate defocus file""" + utils.generateDefocusIMODFileFromObject(ctfTomoSeries, defocusFilePath, + inputTiltSeries=ts, + presentAcqOrders=presentAcqOrders) # --------------------------- INFO functions ------------------------------ def _warnings(self): diff --git a/imod/utils.py b/imod/utils.py index f160617b..cacc3453 100644 --- a/imod/utils.py +++ b/imod/utils.py @@ -633,7 +633,7 @@ def refactorCTFDefocusAstigmatismPhaseShiftCutOnFreqEstimationInfo(ctfInfoIMODTa def generateDefocusIMODFileFromObject(ctfTomoSeries, defocusFilePath, - isRelion=False, inputTiltSeries=None): + isRelion=False, inputTiltSeries=None, presentAcqOrders=None): """ This method takes a ctfTomoSeries object a generate a defocus information file in IMOD formatting containing the same information in the specified location. """ @@ -857,6 +857,8 @@ def generateDefocusIMODFileFromObject(ctfTomoSeries, defocusFilePath, for ti in tiltSeries: ctfTomo = ctfTomoSeries.getCtfTomoFromTi(ti) if ctfTomo: + if presentAcqOrders and ctfTomo.getAcquisitionOrder() not in presentAcqOrders: + continue tiltAngle = ti.getTiltAngle() newLine = ("%d\t%d\t%.2f\t%.2f\t%.1f\t%.1f\t%.2f\n" % ( ind, From 246a922fb98d7e1b14d0abab512db97bd46f16e8 Mon Sep 17 00:00:00 2001 From: JorMaister Date: Wed, 3 Apr 2024 13:41:05 +0200 Subject: [PATCH 15/49] Remove debugging lines --- imod/protocols/protocol_ctfCorrection.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/imod/protocols/protocol_ctfCorrection.py b/imod/protocols/protocol_ctfCorrection.py index fd4375d5..f857be3a 100644 --- a/imod/protocols/protocol_ctfCorrection.py +++ b/imod/protocols/protocol_ctfCorrection.py @@ -137,18 +137,6 @@ def _defineParams(self, form): # -------------------------- INSERT steps functions ----------------------- def _insertAllSteps(self): - # JORGE - import os - fname = "/home/jjimenez/test_JJ.txt" - if os.path.exists(fname): - os.remove(fname) - fjj = open(fname, "a+") - fjj.write('JORGE--------->onDebugMode PID {}'.format(os.getpid())) - fjj.close() - print('JORGE--------->onDebugMode PID {}'.format(os.getpid())) - import time - time.sleep(10) - # JORGE_END nonMatchingTsIds = [] self._initialize() for tsId in self.tsDict.keys(): # Stores the steps serializing the tsId instead of the whole ts object From c111fb3a24fe8ebc185d9065a233f63d6a86709b Mon Sep 17 00:00:00 2001 From: JorMaister Date: Wed, 10 Apr 2024 18:32:21 +0200 Subject: [PATCH 16/49] Protocol refactoring. Tests still to be updated --- imod/protocols/__init__.py | 2 +- .../protocol_applyTransformationMatrix.py | 63 +++-- imod/protocols/protocol_base.py | 29 ++- .../protocol_ctfEstimation_automatic.py | 75 +++--- imod/protocols/protocol_doseFilter.py | 51 ++-- imod/protocols/protocol_fiducialAlignment.py | 235 ++++++------------ imod/protocols/protocol_fiducialModel.py | 149 ++++------- imod/protocols/protocol_tomoReconstruction.py | 77 +++--- ...malization.py => protocol_tsPreprocess.py} | 57 +++-- imod/protocols/protocol_xCorrPrealignment.py | 89 +++---- imod/protocols/protocol_xRaysEraser.py | 60 +++-- imod/tests/test_protocols_imod.py | 2 +- 12 files changed, 351 insertions(+), 538 deletions(-) rename imod/protocols/{protocol_tsNormalization.py => protocol_tsPreprocess.py} (91%) diff --git a/imod/protocols/__init__.py b/imod/protocols/__init__.py index 48590ab7..cf0a04fe 100644 --- a/imod/protocols/__init__.py +++ b/imod/protocols/__init__.py @@ -43,7 +43,7 @@ from .protocol_tomoNormalization import ProtImodTomoNormalization from .protocol_tomoProjection import ProtImodTomoProjection from .protocol_tomoReconstruction import ProtImodTomoReconstruction -from .protocol_tsNormalization import ProtImodTSNormalization +from .protocol_tsPreprocess import ProtImodTsPreprocess from .protocol_xCorrPrealignment import ProtImodXcorrPrealignment from .protocol_xRaysEraser import ProtImodXraysEraser diff --git a/imod/protocols/protocol_applyTransformationMatrix.py b/imod/protocols/protocol_applyTransformationMatrix.py index bacbe4b6..5ec2e2c9 100644 --- a/imod/protocols/protocol_applyTransformationMatrix.py +++ b/imod/protocols/protocol_applyTransformationMatrix.py @@ -34,7 +34,7 @@ from tomo.objects import TiltSeries, TiltImage from .. import Plugin, utils -from .protocol_base import ProtImodBase, EXT_MRCS_TS_EVEN_NAME, EXT_MRCS_TS_ODD_NAME +from .protocol_base import ProtImodBase, EXT_MRCS_TS_EVEN_NAME, EXT_MRCS_TS_ODD_NAME, XF_EXT, ODD, EVEN, MRCS_EXT class ProtImodApplyTransformationMatrix(ProtImodBase): @@ -74,37 +74,34 @@ def _defineParams(self, form): # -------------------------- INSERT steps functions ----------------------- def _insertAllSteps(self): - self._failedTs = [] - - for ts in self.inputSetOfTiltSeries.get(): - self._insertFunctionStep(self.generateTransformFileStep, ts.getObjId()) - self._insertFunctionStep(self.computeAlignmentStep, ts.getObjId()) - self._insertFunctionStep(self.generateOutputStackStep, ts.getObjId()) - self._insertFunctionStep(self.createOutputFailedSet, ts.getObjId()) + self._initialize() + for tsId in self.tsDict.keys(): + self._insertFunctionStep(self.generateTransformFileStep, tsId) + self._insertFunctionStep(self.computeAlignmentStep, tsId) + self._insertFunctionStep(self.generateOutputStackStep, tsId) + self._insertFunctionStep(self.createOutputFailedStep, tsId) self._insertFunctionStep(self.closeOutputSetsStep) # --------------------------- STEPS functions ------------------------------ - def generateTransformFileStep(self, tsObjId): - ts = self.inputSetOfTiltSeries.get()[tsObjId] - tsId = ts.getTsId() - extraPrefix = self._getExtraPath(tsId) - path.makePath(extraPrefix) - utils.formatTransformFile(ts, - os.path.join(extraPrefix, - ts.getFirstItem().parseFileName(extension=".xf"))) + def _initialize(self): + self._failedTs = [] + self.tsDict = {ts.getTsId(): ts.clone(ignoreAttrs=[]) for ts in self.inputSetOfTiltSeries.get()} + + def generateTransformFileStep(self, tsId): + ts = self.tsDict[tsId] + self.genTsPaths(tsId) + utils.formatTransformFile(ts, self.getExtraOutFile(tsId, ext=XF_EXT)) @ProtImodBase.tryExceptDecorator - def computeAlignmentStep(self, tsObjId): - ts = self.inputSetOfTiltSeries.get()[tsObjId] - tsId = ts.getTsId() - extraPrefix = self._getExtraPath(tsId) + def computeAlignmentStep(self, tsId): + ts = self.tsDict[tsId] firstItem = ts.getFirstItem() binning = self.binning.get() paramsAlignment = { 'input': firstItem.getFileName(), - 'output': os.path.join(extraPrefix, firstItem.parseFileName()), - 'xform': os.path.join(extraPrefix, firstItem.parseFileName(extension=".xf")), + 'output': self.getExtraOutFile(tsId), + 'xform': self.getExtraOutFile(tsId, ext=XF_EXT), 'bin': binning, 'imagebinned': 1.0 } @@ -139,19 +136,15 @@ def computeAlignmentStep(self, tsObjId): oddFn = firstItem.getOdd().split('@')[1] evenFn = firstItem.getEven().split('@')[1] paramsAlignment['input'] = oddFn - paramsAlignment['output'] = os.path.join(extraPrefix, tsId+EXT_MRCS_TS_ODD_NAME) + paramsAlignment['output'] = self.getExtraOutFile(tsId, suffix=ODD, ext=MRCS_EXT) Plugin.runImod(self, 'newstack', argsAlignment % paramsAlignment) paramsAlignment['input'] = evenFn - paramsAlignment['output'] = os.path.join(extraPrefix, tsId+EXT_MRCS_TS_EVEN_NAME) + paramsAlignment['output'] = self.getExtraOutFile(tsId, suffix=EVEN, ext=MRCS_EXT) Plugin.runImod(self, 'newstack', argsAlignment % paramsAlignment) - def generateOutputStackStep(self, tsObjId): - - ts = self.inputSetOfTiltSeries.get()[tsObjId] - tsId = ts.getTsId() - extraPrefix = self._getExtraPath(tsId) - - outputLocation = os.path.join(extraPrefix, ts.getFirstItem().parseFileName()) + def generateOutputStackStep(self, tsId): + ts = self.tsDict[tsId] + outputLocation = self.getExtraOutFile(tsId) if os.path.exists(outputLocation): output = self.getOutputInterpolatedSetOfTiltSeries(self.inputSetOfTiltSeries.get()) @@ -181,8 +174,8 @@ def generateOutputStackStep(self, tsObjId): newTi.setAcquisition(acq) newTi.setLocation(index, outputLocation) if self.applyToOddEven(ts): - locationOdd = index, (os.path.join(extraPrefix, tsId + EXT_MRCS_TS_ODD_NAME)) - locationEven = index, (os.path.join(extraPrefix, tsId + EXT_MRCS_TS_EVEN_NAME)) + locationOdd = index, (self.getExtraOutFile(tsId, suffix=ODD, ext=MRCS_EXT)) + locationEven = index, (self.getExtraOutFile(tsId, suffix=EVEN, ext=MRCS_EXT)) newTi.setOddEven([ih.locationToXmipp(locationOdd), ih.locationToXmipp(locationEven)]) else: newTi.setOddEven([]) @@ -201,6 +194,10 @@ def generateOutputStackStep(self, tsObjId): output.write() self._store() + def createOutputFailedStep(self, tsId): + ts = self.tsDict[tsId] + super().createOutputFailedSet(ts) + def closeOutputSetsStep(self): for _, output in self.iterOutputAttributes(): output.setStreamState(Set.STREAM_CLOSED) diff --git a/imod/protocols/protocol_base.py b/imod/protocols/protocol_base.py index c55e76fc..195d428f 100644 --- a/imod/protocols/protocol_base.py +++ b/imod/protocols/protocol_base.py @@ -154,6 +154,16 @@ def convertInputStep(self, tsObjId, generateAngleFile=True, imodInterpolation=Tr self.genAlignmentFiles(ts, generateAngleFile=generateAngleFile, imodInterpolation=imodInterpolation, doSwap=doSwap, oddEven=oddEven, presentAcqOrders=presentAcqOrders) + def applyNewStackBasic(self, ts, outputTsFileName, inputTsFileName, xfFile, doSwap, tsExcludedIndices=None): + argsAlignment, paramsAlignment = self.getBasicNewstackParams(ts, + outputTsFileName, + inputTsFileName=inputTsFileName, + xfFile=xfFile, + firstItem=ts.getFirstItem(), + doSwap=doSwap, + tsExcludedIndices=tsExcludedIndices) + Plugin.runImod(self, 'newstack', argsAlignment % paramsAlignment) + def genAlignmentFiles(self, ts, generateAngleFile=True, imodInterpolation=True, doSwap=False, oddEven=False, presentAcqOrders=None): """ @@ -198,7 +208,10 @@ def genAlignmentFiles(self, ts, generateAngleFile=True, imodInterpolation=True, # Generate the interpolated TS with IMOD's newstack program logger.info("Tilt-series interpolated with IMOD [%s]" % tsId) - tsExcludedIndices = [ti.getIndex() for ti in ts if not ti.getAcquisitionOrder() in presentAcqOrders] + if presentAcqOrders: + tsExcludedIndices = [ti.getIndex() for ti in ts if not ti.getAcquisitionOrder() in presentAcqOrders] + else: + tsExcludedIndices = [ti.getIndex() for ti in ts if not ti.getAcquisitionOrder()] self.applyNewStackBasic(ts, outputTsFileName, inTsFileName, xfFile, doSwap, tsExcludedIndices=tsExcludedIndices) if oddEven: @@ -209,7 +222,7 @@ def genAlignmentFiles(self, ts, generateAngleFile=True, imodInterpolation=True, # If some views were excluded to generate the new stack, a new xfFile containing them should be # generated - if len(ts) != len(presentAcqOrders): + if presentAcqOrders and len(ts) != len(presentAcqOrders): utils.genXfFile(ts, xfFile, presentAcqOrders=presentAcqOrders) else: @@ -232,17 +245,7 @@ def genAlignmentFiles(self, ts, generateAngleFile=True, imodInterpolation=True, if generateAngleFile: logger.info("Generate angle file for the tilt-series [%s]" % tsId) angleFilePath = self.getExtraOutFile(tsId, ext=TLT_EXT) - ts.genTltFile(angleFilePath, presentAcqOrders=presentAcqOrders) - - def applyNewStackBasic(self, ts, outputTsFileName, inputTsFileName, xfFile, doSwap, tsExcludedIndices=None): - argsAlignment, paramsAlignment = self.getBasicNewstackParams(ts, - outputTsFileName, - inputTsFileName=inputTsFileName, - xfFile=xfFile, - firstItem=ts.getFirstItem(), - doSwap=doSwap, - tsExcludedIndices=tsExcludedIndices) - Plugin.runImod(self, 'newstack', argsAlignment % paramsAlignment) + ts.generateTltFile(angleFilePath, presentAcqOrders=presentAcqOrders) @staticmethod def getBasicNewstackParams(ts, outputTsFileName, inputTsFileName=None, diff --git a/imod/protocols/protocol_ctfEstimation_automatic.py b/imod/protocols/protocol_ctfEstimation_automatic.py index 6b57507f..e3bad05e 100644 --- a/imod/protocols/protocol_ctfEstimation_automatic.py +++ b/imod/protocols/protocol_ctfEstimation_automatic.py @@ -32,7 +32,7 @@ import tomo.objects as tomoObj from .. import Plugin, utils -from .protocol_base import ProtImodBase, OUTPUT_CTF_SERIE +from .protocol_base import ProtImodBase, OUTPUT_CTF_SERIE, TLT_EXT class ProtImodAutomaticCtfEstimation(ProtImodBase): @@ -49,6 +49,11 @@ class ProtImodAutomaticCtfEstimation(ProtImodBase): defocusVTolerance = 20 _interactiveMode = False + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.sRate = None + self.acq = None + # -------------------------- DEFINE param functions ----------------------- def _defineParams(self, form): form.addSection('Input') @@ -292,45 +297,41 @@ def _defineParams(self, form): # -------------------------- INSERT steps functions ----------------------- def _insertAllSteps(self): - self._failedTs = [] - - # This assignment is needed to use methods from base class - self.inputSetOfTiltSeries = self._getSetOfTiltSeries() + self._initialize() expDefoci = self.getExpectedDefocus() - - for item in self.inputSet.get(): - tsObjId = item.getObjId() - self._insertFunctionStep(self.convertInputStep, tsObjId) - self._insertFunctionStep(self.ctfEstimation, tsObjId, expDefoci) - self._insertFunctionStep(self.createOutputStep, tsObjId) - self._insertFunctionStep(self.createOutputFailedSet, tsObjId) + for tsId in self.tsDict.keys(): + self._insertFunctionStep(self.convertInputStep, tsId) + self._insertFunctionStep(self.ctfEstimation, tsId, expDefoci) + self._insertFunctionStep(self.createOutputStep, tsId) + self._insertFunctionStep(self.createOutputFailedStep, tsId) self._insertFunctionStep(self.closeOutputSetsStep) # --------------------------- STEPS functions ----------------------------- - def convertInputStep(self, tsObjId, **kwargs): + def _initialize(self): + self._failedTs = [] + tsSet = self._getSetOfTiltSeries() + self.tsDict = {ts.getTsId(): ts.clone(ignoreAttrs=[]) for ts in tsSet} + self.sRate = tsSet.getSamplingRate() + self.acq = tsSet.getAcquisition() + + def convertInputStep(self, tsId, **kwargs): """ Implement the convertStep to cancel interpolation of the tilt series.""" - super().convertInputStep(tsObjId, imodInterpolation=None) + super().convertInputStep(tsId, imodInterpolation=None) @ProtImodBase.tryExceptDecorator - def ctfEstimation(self, tsObjId, expDefoci): + def ctfEstimation(self, tsId, expDefoci): """Run ctfplotter IMOD program""" - ts = self._getTiltSeries(tsObjId) - tsSet = self._getSetOfTiltSeries() - tsId = ts.getTsId() - extraPrefix = self._getExtraPath(tsId) - tmpPrefix = self._getTmpPath(tsId) - - firstTi = ts.getFirstItem() + ts = self.tsDict[tsId] paramsCtfPlotter = { - 'inputStack': os.path.join(tmpPrefix, firstTi.parseFileName()), - 'angleFile': os.path.join(tmpPrefix, firstTi.parseFileName(extension=".tlt")), - 'defocusFile': os.path.join(extraPrefix, firstTi.parseFileName(extension=".defocus")), + 'inputStack': self.getTmpOutFile(tsId), + 'angleFile': self.getExtraOutFile(tsId, ext=TLT_EXT), + 'defocusFile': self.getExtraOutFile(tsId, ext="defocus"), 'axisAngle': ts.getAcquisition().getTiltAxisAngle(), - 'pixelSize': tsSet.getSamplingRate() / 10, - 'voltage': tsSet.getAcquisition().getVoltage(), - 'sphericalAberration': tsSet.getAcquisition().getSphericalAberration(), - 'amplitudeContrast': tsSet.getAcquisition().getAmplitudeContrast(), + 'pixelSize': self.sRate / 10, # nm + 'voltage': self.acq.getVoltage(), + 'sphericalAberration': self.acq.getSphericalAberration(), + 'amplitudeContrast': self.acq.getAmplitudeContrast(), 'defocusTol': self.defocusTol.get(), 'psResolution': 101, 'leftDefTol': self.leftDefTol.get(), @@ -434,13 +435,9 @@ def ctfEstimation(self, tsObjId, expDefoci): Plugin.runImod(self, 'ctfplotter', argsCtfPlotter % paramsCtfPlotter) - def createOutputStep(self, tsObjId, outputSetName=OUTPUT_CTF_SERIE): - ts = self._getTiltSeries(tsObjId) - tsId = ts.getTsId() - - extraPrefix = self._getExtraPath(tsId) - defocusFilePath = os.path.join(extraPrefix, - ts.getFirstItem().parseFileName(extension=".defocus")) + def createOutputStep(self, tsId, outputSetName=OUTPUT_CTF_SERIE): + ts = self.tsDict[tsId] + defocusFilePath = self.getExtraOutFile(tsId, ext="defocus") if os.path.exists(defocusFilePath): output = self.getOutputSetOfCTFTomoSeries(outputSetName) defocusFileFlag = utils.getDefocusFileFlag(defocusFilePath) @@ -448,7 +445,7 @@ def createOutputStep(self, tsObjId, outputSetName=OUTPUT_CTF_SERIE): newCTFTomoSeries = tomoObj.CTFTomoSeries() newCTFTomoSeries.copyInfo(ts) newCTFTomoSeries.setTiltSeries(ts) - newCTFTomoSeries.setObjId(tsObjId) + # newCTFTomoSeries.setObjId(tsObjId) newCTFTomoSeries.setTsId(tsId) newCTFTomoSeries.setIMODDefocusFileFlag(defocusFileFlag) newCTFTomoSeries.setNumberOfEstimationsInRange(None) @@ -464,6 +461,10 @@ def createOutputStep(self, tsObjId, outputSetName=OUTPUT_CTF_SERIE): output.write() self._store() + def createOutputFailedStep(self, tsId): + ts = self.tsDict[tsId] + super().createOutputFailedSet(ts) + def closeOutputSetsStep(self): for _, output in self.iterOutputAttributes(): output.setStreamState(Set.STREAM_CLOSED) diff --git a/imod/protocols/protocol_doseFilter.py b/imod/protocols/protocol_doseFilter.py index fa2d1318..065c67f1 100644 --- a/imod/protocols/protocol_doseFilter.py +++ b/imod/protocols/protocol_doseFilter.py @@ -34,7 +34,7 @@ from pwem.emlib.image import ImageHandler from .. import Plugin, utils -from .protocol_base import ProtImodBase, EXT_MRCS_TS_EVEN_NAME, EXT_MRCS_TS_ODD_NAME +from .protocol_base import ProtImodBase, EXT_MRCS_TS_EVEN_NAME, EXT_MRCS_TS_ODD_NAME, ODD, MRCS_EXT, EVEN SCIPION_IMPORT = 0 FIXED_DOSE = 1 @@ -101,29 +101,26 @@ def _defineParams(self, form): # -------------------------- INSERT steps functions ----------------------- def _insertAllSteps(self): - for ts in self.inputSetOfTiltSeries.get(): - self._insertFunctionStep(self.doseFilterStep, ts.getObjId()) - self._insertFunctionStep(self.createOutputStep, ts.getObjId()) + self._initialize() + for tsId in self.tsDict.keys(): + self._insertFunctionStep(self.doseFilterStep, tsId) + self._insertFunctionStep(self.createOutputStep, tsId) self._insertFunctionStep(self.closeOutputSetsStep) # --------------------------- STEPS functions ----------------------------- - def doseFilterStep(self, tsObjId): - """Apply the dose filter to every tilt series""" - - ts = self.inputSetOfTiltSeries.get()[tsObjId] - tsId = ts.getTsId() + def _initialize(self): + self.tsDict = {ts.getTsId(): ts.clone(ignoreAttrs=[]) for ts in self.inputSetOfTiltSeries.get()} - extraPrefix = self._getExtraPath(tsId) - tmpPrefix = self._getTmpPath(tsId) - - path.makePath(tmpPrefix) - path.makePath(extraPrefix) + def doseFilterStep(self, tsId): + """Apply the dose filter to every tilt series""" + self.genTsPaths(tsId) + ts = self.tsDict[tsId] firstItem = ts.getFirstItem() paramsMtffilter = { 'input': firstItem.getFileName(), - 'output': os.path.join(extraPrefix, firstItem.parseFileName()), + 'output': self.getExtraOutFile(tsId), 'pixsize': ts.getSamplingRate(), 'voltage': ts.getAcquisition().getVoltage(), } @@ -137,9 +134,7 @@ def doseFilterStep(self, tsObjId): argsMtffilter += f"-InitialDose {self.initialDose.get():f} " if self.inputDoseType.get() == SCIPION_IMPORT: - outputDoseFilePath = os.path.join(tmpPrefix, - firstItem.parseFileName(extension=".dose")) - + outputDoseFilePath = self.getExtraOutFile(tsId, ext="dose"), utils.generateDoseFileFromDoseTS(ts, outputDoseFilePath) paramsMtffilter.update({ @@ -162,27 +157,22 @@ def doseFilterStep(self, tsObjId): if self.applyToOddEven(ts): oddFn = firstItem.getOdd().split('@')[1] paramsMtffilter['input'] = oddFn - paramsMtffilter['output'] = os.path.join(extraPrefix, tsId+EXT_MRCS_TS_ODD_NAME) + paramsMtffilter['output'] = self.getExtraOutFile(tsId, suffix=ODD, ext=MRCS_EXT) Plugin.runImod(self, 'mtffilter', argsMtffilter % paramsMtffilter) evenFn = firstItem.getEven().split('@')[1] paramsMtffilter['input'] = evenFn - paramsMtffilter['output'] = os.path.join(extraPrefix, tsId+EXT_MRCS_TS_EVEN_NAME) + paramsMtffilter['output'] = self.getExtraOutFile(tsId, suffix=EVEN, ext=MRCS_EXT) Plugin.runImod(self, 'mtffilter', argsMtffilter % paramsMtffilter) - def createOutputStep(self, tsObjId): + def createOutputStep(self, tsId): """Generate output filtered tilt series""" - ts = self.inputSetOfTiltSeries.get()[tsObjId] - tsId = ts.getTsId() - extraPrefix = self._getExtraPath(tsId) - + ts = self.tsDict[tsId] output = self.getOutputSetOfTiltSeries(self.inputSetOfTiltSeries.get()) newTs = tomoObj.TiltSeries(tsId=tsId) newTs.copyInfo(ts) - output.append(newTs) - ih = ImageHandler() for index, tiltImage in enumerate(ts): @@ -190,14 +180,13 @@ def createOutputStep(self, tsObjId): newTi.copyInfo(tiltImage, copyId=True, copyTM=True) newTi.setAcquisition(tiltImage.getAcquisition()) if self.applyToOddEven(ts): - locationOdd = index + 1, (os.path.join(extraPrefix, tsId+EXT_MRCS_TS_ODD_NAME)) - locationEven = index + 1, (os.path.join(extraPrefix, tsId+EXT_MRCS_TS_EVEN_NAME)) + locationOdd = index + 1, self.getExtraOutFile(tsId, suffix=ODD, ext=MRCS_EXT) + locationEven = index + 1, self.getExtraOutFile(tsId, suffix=EVEN, ext=MRCS_EXT) newTi.setOddEven([ih.locationToXmipp(locationOdd), ih.locationToXmipp(locationEven)]) else: newTi.setOddEven([]) - locationTi = index + 1, (os.path.join(extraPrefix, - tiltImage.parseFileName())) + locationTi = index + 1, self.getExtraOutFile(tsId) newTi.setLocation(locationTi) newTs.append(newTi) newTs.update(newTi) diff --git a/imod/protocols/protocol_fiducialAlignment.py b/imod/protocols/protocol_fiducialAlignment.py index f261fbc0..18666f95 100644 --- a/imod/protocols/protocol_fiducialAlignment.py +++ b/imod/protocols/protocol_fiducialAlignment.py @@ -36,7 +36,7 @@ TiltSeries, TiltSeriesCoordinate) from .. import Plugin, utils -from .protocol_base import ProtImodBase +from .protocol_base import ProtImodBase, TLT_EXT, XF_EXT class ProtImodFiducialAlignment(ProtImodBase): @@ -236,80 +236,59 @@ def _defineParams(self, form): # -------------------------- INSERT steps functions ----------------------- def _insertAllSteps(self): - self.inputSetOfTiltSeries = self.inputSetOfLandmarkModels.get().getSetOfTiltSeries(pointer=True) - - tsIds = self.inputSetOfLandmarkModels.get().aggregate(["COUNT"], "_tsId", ["_tsId"]) - tsIds = set([d['_tsId'] for d in tsIds]) - - tsIdsDict = {ts.getTsId(): ts.clone(ignoreAttrs=[]) for ts in - self.inputSetOfTiltSeries.get() if - ts.getTsId() in tsIds} - - self._failedTs = [] - + self._initialize() for lm in self.inputSetOfLandmarkModels.get(): lmTsId = lm.getTsId() self.fiducialDiameterPixel = lm.getSize() - - tsObjId = tsIdsDict[lmTsId].getObjId() - self._insertFunctionStep(self.convertInputStep, tsObjId) - self._insertFunctionStep(self.computeFiducialAlignmentStep, tsObjId) - self._insertFunctionStep(self.translateFiducialPointModelStep, tsObjId) - self._insertFunctionStep(self.computeOutputStackStep, tsObjId) + self._insertFunctionStep(self.convertInputStep, lmTsId) + self._insertFunctionStep(self.computeFiducialAlignmentStep, lmTsId) + self._insertFunctionStep(self.translateFiducialPointModelStep, lmTsId) + self._insertFunctionStep(self.computeOutputStackStep, lmTsId) if self.computeAlignment.get() == 0 or self.eraseGoldBeads.get() == 0: self._insertFunctionStep(self.computeOutputInterpolatedStackStep, - tsObjId) + lmTsId) if self.eraseGoldBeads.get() == 0: - self._insertFunctionStep(self.eraseGoldBeadsStep, tsObjId) + self._insertFunctionStep(self.eraseGoldBeadsStep, lmTsId) - self._insertFunctionStep(self.computeOutputModelsStep, tsObjId) - self._insertFunctionStep(self.createOutputFailedSet, tsObjId) + self._insertFunctionStep(self.computeOutputModelsStep, lmTsId) + self._insertFunctionStep(self.createOutputFailedStep, lmTsId) self._insertFunctionStep(self.createOutputStep) # --------------------------- STEPS functions ----------------------------- - @ProtImodBase.tryExceptDecorator - def computeFiducialAlignmentStep(self, tsObjId): - ts = self.inputSetOfTiltSeries.get()[tsObjId] - tsId = ts.getTsId() + def _initialize(self): + self.inputSetOfTiltSeries = self.inputSetOfLandmarkModels.get().getSetOfTiltSeries(pointer=True) - lm = self.inputSetOfLandmarkModels.get().getLandmarkModelFromTsId(tsId=tsId) + tsIds = self.inputSetOfLandmarkModels.get().aggregate(["COUNT"], "_tsId", ["_tsId"]) + tsIds = set([d['_tsId'] for d in tsIds]) - extraPrefix = self._getExtraPath(tsId) - tmpPrefix = self._getTmpPath(tsId) + self.tsDict = {ts.getTsId(): ts.clone(ignoreAttrs=[]) for ts in + self.inputSetOfTiltSeries.get() if + ts.getTsId() in tsIds} - firstItem = ts.getFirstItem() + self._failedTs = [] + + @ProtImodBase.tryExceptDecorator + def computeFiducialAlignmentStep(self, tsId): + ts = self.tsDict[tsId] + lm = self.inputSetOfLandmarkModels.get().getLandmarkModelFromTsId(tsId=tsId) paramsTiltAlign = { 'modelFile': lm.getModelName(), - 'imageFile': os.path.join(tmpPrefix, firstItem.parseFileName()), + 'imageFile': self.getTmpOutFile(tsId), 'imagesAreBinned': 1, 'unbinnedPixelSize': ts.getSamplingRate() / 10, - 'outputModelFile': os.path.join(extraPrefix, - firstItem.parseFileName(suffix="_fidxyz", - extension=".mod")), - 'outputResidualFile': os.path.join(extraPrefix, - firstItem.parseFileName(suffix="_resid", - extension=".txt")), - 'outputFidXYZFile': os.path.join(extraPrefix, - firstItem.parseFileName(suffix="_fid", - extension=".xyz")), - 'outputTiltFile': os.path.join(extraPrefix, - firstItem.parseFileName(suffix="_interpolated", - extension=".tlt")), - 'outputXAxisTiltFile': os.path.join(extraPrefix, - firstItem.parseFileName(extension=".xtilt")), - 'outputTransformFile': os.path.join(extraPrefix, - firstItem.parseFileName(suffix="_fid", - extension=".xf")), - 'outputFilledInModel': os.path.join(extraPrefix, - firstItem.parseFileName(suffix="_noGaps", - extension=".fid")), + 'outputModelFile': self.getExtraOutFile(tsId, suffix="fidxyz", ext="mod"), + 'outputResidualFile': self.getExtraOutFile(tsId, suffix="resid", ext="txt"), + 'outputFidXYZFile': self.getExtraOutFile(tsId, suffix="fid", ext="xyz"), + 'outputTiltFile': self.getExtraOutFile(tsId, suffix="interpolated", ext=TLT_EXT), + 'outputXAxisTiltFile': self.getExtraOutFile(tsId, ext="xtilt"), + 'outputTransformFile': self.getExtraOutFile(tsId, suffix="fid", ext="xf"), + 'outputFilledInModel': self.getExtraOutFile(tsId, suffix="noGaps", ext="fid"), 'rotationAngle': ts.getAcquisition().getTiltAxisAngle(), - 'tiltFile': os.path.join(tmpPrefix, - firstItem.parseFileName(extension=".tlt")), + 'tiltFile': self.getExtraOutFile(tsId, ext=TLT_EXT), 'angleOffset': 0.0, 'rotOption': self.getRotationType(), 'rotDefaultGrouping': self.groupRotationSize.get(), @@ -334,9 +313,7 @@ def computeFiducialAlignmentStep(self, tsObjId): 'axisZShift': 0.0, 'shiftZFromOriginal': 1, 'localAlignments': 0, - 'outputLocalFile': os.path.join(extraPrefix, - firstItem.parseFileName(suffix="_local", - extension=".xf")), + 'outputLocalFile': self.getExtraOutFile(tsId, suffix="local", ext=XF_EXT), 'targetPatchSizeXandY': '700,700', 'minSizeOrOverlapXandY': '0.5,0.5', 'minFidsTotalAndEachSurface': '8,3', @@ -353,7 +330,7 @@ def computeFiducialAlignmentStep(self, tsObjId): 'localXStretchDefaultGrouping': 7, 'localSkewOption': 0, 'localSkewDefaultGrouping': 11, - 'outputTiltAlignFileText': os.path.join(extraPrefix, "align.log"), + 'outputTiltAlignFileText': self._getExtraPath("align.log"), } argsTiltAlign = "-ModelFile %(modelFile)s " \ @@ -420,28 +397,16 @@ def computeFiducialAlignmentStep(self, tsObjId): argsTiltAlign += "2>&1 | tee %(outputTiltAlignFileText)s " Plugin.runImod(self, 'tiltalign', argsTiltAlign % paramsTiltAlign) - Plugin.runImod(self, 'alignlog', '-s > taSolution.log', cwd=extraPrefix) + Plugin.runImod(self, 'alignlog', '-s > taSolution.log', cwd=self._getExtraPath()) @ProtImodBase.tryExceptDecorator - def translateFiducialPointModelStep(self, tsObjId): - ts = self.inputSetOfTiltSeries.get()[tsObjId] - tsId = ts.getTsId() - - extraPrefix = self._getExtraPath(tsId) - - firstItem = ts.getFirstItem() - + def translateFiducialPointModelStep(self, tsId): # Check that previous steps have been completed satisfactorily - if os.path.exists(os.path.join(extraPrefix, - firstItem.parseFileName(suffix="_noGaps", - extension=".fid"))): + noGapsFid = self.getExtraOutFile(tsId, suffix="noGaps", ext="fid") + if os.path.exists(noGapsFid): paramsNoGapModel2Point = { - 'inputFile': os.path.join(extraPrefix, - firstItem.parseFileName(suffix="_noGaps", - extension=".fid")), - 'outputFile': os.path.join(extraPrefix, - firstItem.parseFileName(suffix="_noGaps_fid", - extension=".txt")) + 'inputFile': noGapsFid, + 'outputFile': self.getExtraOutFile(tsId, suffix="noGaps_fid", ext="txt") } argsNoGapModel2Point = "-InputFile %(inputFile)s " \ "-OutputFile %(outputFile)s" @@ -449,32 +414,15 @@ def translateFiducialPointModelStep(self, tsObjId): Plugin.runImod(self, 'model2point', argsNoGapModel2Point % paramsNoGapModel2Point) @ProtImodBase.tryExceptDecorator - def computeOutputStackStep(self, tsObjId): - ts = self.inputSetOfTiltSeries.get()[tsObjId] - tsId = ts.getTsId() - - extraPrefix = self._getExtraPath(tsId) - - firstItem = ts.getFirstItem() + def computeOutputStackStep(self, tsId): + ts = self.tsDict[tsId] # Check that previous steps have been completed satisfactorily - tmpFileName = os.path.join(extraPrefix, - firstItem.parseFileName(suffix="_fid", - extension=".xf")) - if os.path.exists(tmpFileName) and os.stat(tmpFileName).st_size != 0: - tltFilePath = os.path.join( - extraPrefix, - firstItem.parseFileName(suffix="_interpolated", extension=".tlt") - ) + transformationMatricesFilePath = self.getExtraOutFile(tsId, suffix="fid", ext=XF_EXT) + if os.path.exists(transformationMatricesFilePath) and os.stat(transformationMatricesFilePath).st_size != 0: + tltFilePath = self.getExtraOutFile(tsId, suffix="interpolated", ext=TLT_EXT) tltList = utils.formatAngleList(tltFilePath) - - transformationMatricesFilePath = os.path.join( - extraPrefix, - firstItem.parseFileName(suffix="_fid", extension=".xf") - ) - newTransformationMatricesList = utils.formatTransformationMatrix(transformationMatricesFilePath) - output = self.getOutputSetOfTiltSeries(self.inputSetOfTiltSeries.get()) newTs = TiltSeries(tsId=tsId) newTs.copyInfo(ts) @@ -514,30 +462,23 @@ def computeOutputStackStep(self, tsObjId): else: raise FileNotFoundError( "Error (computeOutputStackStep): \n Imod output file " - "%s does not exist or it is empty" % tmpFileName) + "%s does not exist or it is empty" % transformationMatricesFilePath) @ProtImodBase.tryExceptDecorator - def computeOutputInterpolatedStackStep(self, tsObjId): - tsIn = self.inputSetOfTiltSeries.get()[tsObjId] + def computeOutputInterpolatedStackStep(self, tsId): + tsIn = self.tsDict[tsId] tsId = tsIn.getTsId() - - extraPrefix = self._getExtraPath(tsId) - tmpPrefix = self._getTmpPath(tsId) - firstItem = tsIn.getFirstItem() # Check that previous steps have been completed satisfactorily - tmpFileName = os.path.join(extraPrefix, - firstItem.parseFileName(suffix="_fid", - extension=".xf")) + tmpFileName = self.getExtraOutFile(tsId, suffix="fid", ext=XF_EXT) if os.path.exists(tmpFileName) and os.stat(tmpFileName).st_size != 0: output = self.getOutputInterpolatedSetOfTiltSeries(self.inputSetOfTiltSeries.get()) paramsAlignment = { - 'input': os.path.join(tmpPrefix, firstItem.parseFileName()), - 'output': os.path.join(extraPrefix, firstItem.parseFileName()), - 'xform': os.path.join(extraPrefix, firstItem.parseFileName(suffix="_fid", - extension=".xf")), + 'input': self.getTmpOutFile(tsId), + 'output': self.getExtraOutFile(tsId), + 'xform': tmpFileName, 'bin': self.binning.get(), 'imagebinned': 1.0} @@ -570,11 +511,7 @@ def computeOutputInterpolatedStackStep(self, tsObjId): newTs.setInterpolated(True) output.append(newTs) - tltFilePath = os.path.join( - extraPrefix, - firstItem.parseFileName(suffix="_interpolated", extension=".tlt") - ) - + tltFilePath = self.getExtraOutFile(tsId, suffix="interpolated", ext=TLT_EXT) tltList = utils.formatAngleList(tltFilePath) if self.binning > 1: @@ -584,7 +521,7 @@ def computeOutputInterpolatedStackStep(self, tsObjId): newTi = TiltImage() newTi.copyInfo(tiltImage, copyId=True, copyTM=False) newTi.setAcquisition(tiltImage.getAcquisition()) - newTi.setLocation(index + 1, os.path.join(extraPrefix, tiltImage.parseFileName())) + newTi.setLocation(index + 1, self.getExtraOutFile(tsId)) newTi.setTiltAngle(float(tltList[index])) if self.binning > 1: newTi.setSamplingRate(tiltImage.getSamplingRate() * self.binning.get()) @@ -604,22 +541,15 @@ def computeOutputInterpolatedStackStep(self, tsObjId): "Imod output file %s does not exist or it is empty" % tmpFileName) @ProtImodBase.tryExceptDecorator - def eraseGoldBeadsStep(self, tsObjId): - ts = self.inputSetOfTiltSeries.get()[tsObjId] + def eraseGoldBeadsStep(self, tsId): + ts = self.tsDict[tsId] tsId = ts.getTsId() - extraPrefix = self._getExtraPath(tsId) - tmpPrefix = self._getTmpPath(tsId) - - firstItem = ts.getFirstItem() - # Erase gold beads on aligned stack paramsCcderaser = { - 'inputFile': os.path.join(tmpPrefix, firstItem.parseFileName()), - 'outputFile': os.path.join(extraPrefix, firstItem.parseFileName()), - 'modelFile': os.path.join(extraPrefix, - firstItem.parseFileName(suffix="_noGaps", - extension=".fid")), + 'inputFile': self.getTmpOutFile(tsId), + 'outputFile': self.getExtraOutFile(tsId), + 'modelFile': self.getExtraOutFile(tsId, suffix="noGaps", ext="fid"), 'betterRadius': self.betterRadius.get() / 2, 'polynomialOrder': 0, 'circleObjects': "/" @@ -639,44 +569,19 @@ def eraseGoldBeadsStep(self, tsObjId): Plugin.runImod(self, 'ccderaser', argsCcderaser % paramsCcderaser) @ProtImodBase.tryExceptDecorator - def computeOutputModelsStep(self, tsObjId): - ts = self.inputSetOfTiltSeries.get()[tsObjId] + def computeOutputModelsStep(self, tsId): + ts = self.tsDict[tsId] tsId = ts.getTsId() - extraPrefix = self._getExtraPath(tsId) - - firstItem = ts.getFirstItem() # Create the output set of landmark models with no gaps - if os.path.exists( - os.path.join(extraPrefix, - ts.getFirstItem().parseFileName(suffix="_noGaps_fid", - extension=".txt"))): - + fiducialNoGapFilePath = self.getExtraOutFile(tsId, suffix="noGaps_fid", ext='txt') + if os.path.exists(fiducialNoGapFilePath): output = self.getOutputFiducialModelNoGaps() - output.setSetOfTiltSeries(self.inputSetOfTiltSeries.get()) - - fiducialNoGapFilePath = os.path.join( - extraPrefix, - firstItem.parseFileName(suffix="_noGaps_fid", extension=".txt") - ) - fiducialNoGapList = utils.formatFiducialList(fiducialNoGapFilePath) - - fiducialModelNoGapPath = os.path.join( - extraPrefix, - firstItem.parseFileName(suffix="_noGaps", extension=".fid") - ) - - landmarkModelNoGapsFilePath = os.path.join( - extraPrefix, - firstItem.parseFileName(suffix="_noGaps", extension=".sfid") - ) - - landmarkModelNoGapsResidPath = os.path.join( - extraPrefix, - firstItem.parseFileName(suffix="_resid", extension=".txt") - ) + fiducialModelNoGapPath = self.getExtraOutFile(tsId, suffix="noGaps", ext="fid") + landmarkModelNoGapsFilePath = self.getExtraOutFile(tsId, suffix="noGaps", ext="sfid") + landmarkModelNoGapsResidPath = self.getExtraOutFile(tsId, suffix="resid", ext="txt") fiducialNoGapsResidList = utils.formatFiducialResidList(landmarkModelNoGapsResidPath) @@ -719,9 +624,7 @@ def computeOutputModelsStep(self, tsObjId): output.write() # Create the output set of 3D coordinates - coordFilePath = os.path.join(extraPrefix, - firstItem.parseFileName(suffix="_fid", - extension=".xyz")) + coordFilePath = self.getExtraOutFile(tsId, suffix="fid", ext="xyz") if os.path.exists(coordFilePath): @@ -755,6 +658,10 @@ def createOutputStep(self): self._store() + def createOutputFailedStep(self, tsId): + ts = self.tsDict[tsId] + super().createOutputFailedSet(ts) + # --------------------------- UTILS functions ----------------------------- def getRotationType(self): if self.rotationSolutionType.get() == 0: diff --git a/imod/protocols/protocol_fiducialModel.py b/imod/protocols/protocol_fiducialModel.py index 714f6ed4..3172328b 100644 --- a/imod/protocols/protocol_fiducialModel.py +++ b/imod/protocols/protocol_fiducialModel.py @@ -33,7 +33,7 @@ import tomo.objects as tomoObj from .. import Plugin, utils -from .protocol_base import ProtImodBase +from .protocol_base import ProtImodBase, TLT_EXT, XF_EXT class ProtImodFiducialModel(ProtImodBase): @@ -136,43 +136,35 @@ def _defineParams(self, form): # -------------------------- INSERT steps functions ----------------------- def _insertAllSteps(self): - self._failedTs = [] - - for ts in self.inputSetOfTiltSeries.get(): - tsObjId = ts.getObjId() - self._insertFunctionStep(self.convertInputStep, tsObjId) - self._insertFunctionStep(self.generateTrackComStep, tsObjId) - self._insertFunctionStep(self.generateFiducialSeedStep, tsObjId) - self._insertFunctionStep(self.generateFiducialModelStep, tsObjId) - self._insertFunctionStep(self.translateFiducialPointModelStep, tsObjId) - self._insertFunctionStep(self.computeOutputModelsStep, tsObjId) - self._insertFunctionStep(self.createOutputFailedSet, tsObjId) + self._initialize() + for tsId in self.tsDict.keys(): + self._insertFunctionStep(self.convertInputStep, tsId) + self._insertFunctionStep(self.generateTrackComStep, tsId) + self._insertFunctionStep(self.generateFiducialSeedStep, tsId) + self._insertFunctionStep(self.generateFiducialModelStep, tsId) + self._insertFunctionStep(self.translateFiducialPointModelStep, tsId) + self._insertFunctionStep(self.computeOutputModelsStep, tsId) + self._insertFunctionStep(self.createOutputFailedSetStep, tsId) self._insertFunctionStep(self.createOutputStep) # --------------------------- STEPS functions ----------------------------- - def generateTrackComStep(self, tsObjId): - ts = self._getTiltSeries(tsObjId) - tsId = ts.getTsId() - - extraPrefix = self._getExtraPath(tsId) - tmpPrefix = self._getTmpPath(tsId) - - firstItem = ts.getFirstItem() + def _initialize(self): + self._failedTs = [] + self.tsDict = {ts.getTsId(): ts.clone(ignoreAttrs=[]) for ts in self.inputSetOfTiltSeries.get()} + def generateTrackComStep(self, tsId): + ts = self.tsDict[tsId] fiducialDiameterPixel = self.fiducialDiameter.get() / (self.inputSetOfTiltSeries.get().getSamplingRate() / 10) boxSizeXandY = max(3.3 * fiducialDiameterPixel + 2, 2 * fiducialDiameterPixel + 20, 32) boxSizeXandY = min(512, 2 * int(boxSizeXandY / 2)) scaling = fiducialDiameterPixel / 12.5 if fiducialDiameterPixel > 12.5 else 1 paramsDict = { - 'imageFile': os.path.join(tmpPrefix, firstItem.parseFileName()), - 'inputSeedModel': os.path.join(extraPrefix, - firstItem.parseFileName(extension=".seed")), - 'outputModel': os.path.join(extraPrefix, - firstItem.parseFileName(suffix="_gaps", - extension=".fid")), - 'tiltFile': os.path.join(tmpPrefix, firstItem.parseFileName(extension=".tlt")), + 'imageFile': self.getTmpOutFile(tsId), + 'inputSeedModel': self.getExtraOutFile(tsId, ext="seed"), + 'outputModel': self.getExtraOutFile(tsId, suffix="gaps", ext="fid"), + 'tiltFile': self.getExtraOutFile(tsId, ext=TLT_EXT), 'rotationAngle': ts.getAcquisition().getTiltAxisAngle(), 'fiducialDiameter': fiducialDiameterPixel, 'samplingRate': self.inputSetOfTiltSeries.get().getSamplingRate() / 10, @@ -188,16 +180,9 @@ def generateTrackComStep(self, tsObjId): self.translateTrackCom(ts, paramsDict) @ProtImodBase.tryExceptDecorator - def generateFiducialSeedStep(self, tsObjId): - ts = self._getTiltSeries(tsObjId) - tsId = ts.getTsId() - - extraPrefix = self._getExtraPath(tsId) - + def generateFiducialSeedStep(self, tsId): paramsAutofidseed = { - 'trackCommandFile': os.path.join(extraPrefix, - ts.getFirstItem().parseFileName(suffix="_track", - extension=".com")), + 'trackCommandFile': self.getExtraOutFile(tsId, suffix="track", ext="com"), 'minSpacing': 0.85, 'peakStorageFraction': 1.0, 'targetNumberOfBeads': self.numberFiducial.get(), @@ -222,13 +207,8 @@ def generateFiducialSeedStep(self, tsObjId): path.moveFile("autofidseed.info", self._getExtraPath(tsId)) @ProtImodBase.tryExceptDecorator - def generateFiducialModelStep(self, tsObjId): - ts = self._getTiltSeries(tsObjId) - tsId = ts.getTsId() - - extraPrefix = self._getExtraPath(tsId) - tmpPrefix = self._getTmpPath(tsId) - + def generateFiducialModelStep(self, tsId): + ts = self.tsDict[tsId] firstItem = ts.getFirstItem() fiducialDiameterPixel = self.fiducialDiameter.get() / (self.inputSetOfTiltSeries.get().getSamplingRate() / 10) @@ -237,15 +217,11 @@ def generateFiducialModelStep(self, tsObjId): scaling = fiducialDiameterPixel / 12.5 if fiducialDiameterPixel > 12.5 else 1 paramsBeadtrack = { - 'inputSeedModel': os.path.join(extraPrefix, - firstItem.parseFileName(extension=".seed")), - 'outputModel': os.path.join(extraPrefix, - firstItem.parseFileName(suffix="_gaps", - extension=".fid")), - 'imageFile': os.path.join(tmpPrefix, firstItem.parseFileName()), + 'inputSeedModel': self.getExtraOutFile(tsId, ext="seed"), + 'outputModel': self.getExtraOutFile(tsId, suffix="gaps", ext="fid"), + 'imageFile': self.getTmpOutFile(tsId), 'imagesAreBinned': 1, - 'tiltFile': os.path.join(tmpPrefix, - firstItem.parseFileName(extension=".tlt")), + 'tiltFile': self.getExtraOutFile(tsId, ext=TLT_EXT), 'tiltDefaultGrouping': 7, 'magDefaultGrouping': 5, 'rotDefaultGrouping': 1, @@ -314,8 +290,7 @@ def generateFiducialModelStep(self, tsObjId): argsBeadtrack += f"-SkipViews {','.join(excludedViews)} " if firstItem.hasTransform(): - XfFileName = os.path.join(tmpPrefix, - firstItem.parseFileName(extension=".xf")) + XfFileName = self.getExtraOutFile(tsId, ext=XF_EXT) argsBeadtrack += f"-prexf {XfFileName} " Plugin.runImod(self, 'beadtrack', argsBeadtrack % paramsBeadtrack) @@ -323,74 +298,41 @@ def generateFiducialModelStep(self, tsObjId): if self.doTrackWithModel: # repeat tracking with the current model as seed path.copyFile(paramsBeadtrack['inputSeedModel'], - os.path.join(extraPrefix, - firstItem.parseFileName(suffix="_orig", - extension=".seed"))) + self.getExtraOutFile(tsId, suffix="orig", ext="seed")) path.moveFile(paramsBeadtrack['outputModel'], paramsBeadtrack['inputSeedModel']) Plugin.runImod(self, 'beadtrack', argsBeadtrack % paramsBeadtrack) - def translateFiducialPointModelStep(self, tsObjId): - ts = self._getTiltSeries(tsObjId) - tsId = ts.getTsId() - - extraPrefix = self._getExtraPath(tsId) - - firstItem = ts.getFirstItem() + def translateFiducialPointModelStep(self, tsId): + ts = self.tsDict[tsId] # Check that previous steps have been completed satisfactorily - if os.path.exists(os.path.join(extraPrefix, - firstItem.parseFileName(suffix="_gaps", - extension=".fid"))): + gapsFidFile = self.getExtraOutFile(tsId, suffix='gaps', ext='fid') + if os.path.exists(gapsFidFile): paramsGapModel2Point = { - 'inputFile': os.path.join(extraPrefix, - firstItem.parseFileName(suffix="_gaps", - extension=".fid")), - 'outputFile': os.path.join(extraPrefix, - firstItem.parseFileName(suffix="_gaps_fid", - extension=".txt")) + 'inputFile': gapsFidFile, + 'outputFile': self.getExtraOutFile(tsId, suffix="gaps_fid", ext="txt") } argsGapModel2Point = "-InputFile %(inputFile)s " \ "-OutputFile %(outputFile)s" Plugin.runImod(self, 'model2point', argsGapModel2Point % paramsGapModel2Point) - def computeOutputModelsStep(self, tsObjId): - ts = self._getTiltSeries(tsObjId) - tsId = ts.getTsId() - - extraPrefix = self._getExtraPath(tsId) - - firstItem = ts.getFirstItem() + def computeOutputModelsStep(self, tsId): + ts = self.tsDict[tsId] # Create the output set of landmark models with gaps # Check that previous steps have been completed satisfactorily - if os.path.exists( - os.path.join(extraPrefix, - firstItem.parseFileName(suffix="_gaps", - extension=".fid"))): - + fiducialModelGapPath = self.getExtraOutFile(tsId, suffix='gaps', ext='fid') + if os.path.exists(fiducialModelGapPath): output = self.getOutputFiducialModelGaps() - - landmarkModelGapsFilePath = os.path.join( - extraPrefix, - firstItem.parseFileName(suffix="_gaps", extension=".sfid") - ) - - fiducialModelGapPath = os.path.join( - extraPrefix, - firstItem.parseFileName(suffix="_gaps", extension=".fid") - ) - - fiducialModelGapTxtPath = os.path.join( - extraPrefix, - firstItem.parseFileName(suffix="_gaps_fid", extension=".txt") - ) + landmarkModelGapsFilePath = self.getExtraOutFile(tsId, suffix='gaps', ext='sfid') + fiducialModelGapTxtPath = self.getExtraOutFile(tsId, suffix="gaps_fid", ext="txt") fiducialGapList = utils.formatFiducialList(fiducialModelGapTxtPath) fiducialDiameterPixel = self.fiducialDiameter.get() / ( - self.inputSetOfTiltSeries.get().getSamplingRate() / 10) + self.inputSetOfTiltSeries.get().getSamplingRate() / 10) landmarkModelGaps = tomoObj.LandmarkModel(tsId=tsId, tiltSeriesPointer=ts, @@ -429,6 +371,10 @@ def createOutputStep(self): self._store() + def createOutputFailedSetStep(self, tsId): + ts = self.tsDict[tsId] + super().createOutputFailedSet(ts) + # --------------------------- UTILS functions ----------------------------- def translateTrackCom(self, ts, paramsDict): tsId = ts.getTsId() @@ -515,8 +461,7 @@ def translateTrackCom(self, ts, paramsDict): template += f"SkipViews {','.join(excludedViews)}" if firstItem.hasTransform(): - XfFileName = os.path.join(tmpPrefix, - firstItem.parseFileName(extension=".xf")) + XfFileName = self.getExtraOutFile(tsId, ext=XF_EXT) template += f"PrealignTransformFile {XfFileName}" with open(trackFilePath, 'w') as f: diff --git a/imod/protocols/protocol_tomoReconstruction.py b/imod/protocols/protocol_tomoReconstruction.py index af84c07d..53725667 100644 --- a/imod/protocols/protocol_tomoReconstruction.py +++ b/imod/protocols/protocol_tomoReconstruction.py @@ -33,7 +33,7 @@ from .. import Plugin from .protocol_base import (ProtImodBase, EXT_MRC_ODD_NAME, EXT_MRC_EVEN_NAME, - EXT_MRCS_TS_EVEN_NAME, EXT_MRCS_TS_ODD_NAME) + EXT_MRCS_TS_EVEN_NAME, EXT_MRCS_TS_ODD_NAME, TLT_EXT, ODD, MRCS_EXT, EVEN) class ProtImodTomoReconstruction(ProtImodBase): @@ -177,35 +177,31 @@ def _defineParams(self, form): # -------------------------- INSERT steps functions ----------------------- def _insertAllSteps(self): - self._failedTs = [] - - for ts in self.inputSetOfTiltSeries.get(): - self._insertFunctionStep(self.convertInputStep, ts.getObjId()) - self._insertFunctionStep(self.computeReconstructionStep, ts.getObjId()) - self._insertFunctionStep(self.createOutputStep, ts.getObjId()) - self._insertFunctionStep(self.createOutputFailedSet, ts.getObjId()) + self._initialize() + for tsId in self.tsDict.keys(): + self._insertFunctionStep(self.convertInputStep, tsId) + self._insertFunctionStep(self.computeReconstructionStep, tsId) + self._insertFunctionStep(self.createOutputStep, tsId) + self._insertFunctionStep(self.createOutputFailedStep, tsId) self._insertFunctionStep(self.closeOutputSetsStep) # --------------------------- STEPS functions ----------------------------- - def convertInputStep(self, tsObjId, **kwargs): + def _initialize(self): + self._failedTs = [] + self.tsDict = {ts.getTsId(): ts.clone(ignoreAttrs=[]) for ts in self.inputSetOfTiltSeries.get()} + + def convertInputStep(self, tsId, **kwargs): # Considering swapXY is required to make tilt axis vertical oddEvenFlag = self.applyToOddEven(self.inputSetOfTiltSeries.get()) - super().convertInputStep(tsObjId, doSwap=True, oddEven=oddEvenFlag) + super().convertInputStep(tsId, doSwap=True, oddEven=oddEvenFlag) @ProtImodBase.tryExceptDecorator - def computeReconstructionStep(self, tsObjId): - ts = self.inputSetOfTiltSeries.get()[tsObjId] - tsId = ts.getTsId() - - firstItem = ts.getFirstItem() - - extraPrefix = self._getExtraPath(tsId) - tmpPrefix = self._getTmpPath(tsId) - + def computeReconstructionStep(self, tsId): + ts = self.tsDict[tsId] paramsTilt = { - 'InputProjections': os.path.join(tmpPrefix, firstItem.parseFileName()), - 'OutputFile': os.path.join(tmpPrefix, firstItem.parseFileName(extension=".rec")), - 'TiltFile': os.path.join(tmpPrefix, firstItem.parseFileName(extension=".tlt")), + 'InputProjections': self.getTmpOutFile(tsId), + 'OutputFile': self.getTmpOutFile(tsId, ext="rec"), + 'TiltFile': self.getExtraOutFile(tsId, ext=TLT_EXT), 'Thickness': self.tomoThickness.get(), 'FalloffIsTrueSigma': 1, 'Radial': str(self.radialFirstParameter.get()) + "," + str(self.radialSecondParameter.get()), @@ -255,21 +251,19 @@ def getArgs(): oddEvenTmp = [[], []] if self.applyToOddEven(ts): - oddFn = os.path.join(tmpPrefix, tsId + EXT_MRCS_TS_ODD_NAME) - paramsTilt['InputProjections'] = oddFn - oddEvenTmp[0] = os.path.join(tmpPrefix, firstItem.parseFileName(extension="_odd.rec")) + paramsTilt['InputProjections'] = self.getExtraOutFile(tsId, suffix=ODD, ext=MRCS_EXT) + oddEvenTmp[0] = self.getExtraOutFile(tsId, suffix=ODD, ext="rec") paramsTilt['OutputFile'] = oddEvenTmp[0] Plugin.runImod(self, 'tilt', argsTilt % paramsTilt) - evenFn = os.path.join(tmpPrefix, tsId + EXT_MRCS_TS_EVEN_NAME) - paramsTilt['InputProjections'] = evenFn - oddEvenTmp[1] = os.path.join(tmpPrefix, firstItem.parseFileName(extension="_even.rec")) + paramsTilt['InputProjections'] = self.getExtraOutFile(tsId, suffix=EVEN, ext=MRCS_EXT) + oddEvenTmp[1] = self.getExtraOutFile(tsId, suffix=EVEN, ext="rec") paramsTilt['OutputFile'] = oddEvenTmp[1] Plugin.runImod(self, 'tilt', argsTilt % paramsTilt) paramsTrimVol = { - 'input': os.path.join(tmpPrefix, firstItem.parseFileName(extension=".rec")), - 'output': os.path.join(extraPrefix, firstItem.parseFileName(extension=".mrc")), + 'input': self.getTmpOutFile(tsId, ext="rec"), + 'output': self.getExtraOutFile(tsId), 'options': getArgs() } @@ -281,21 +275,16 @@ def getArgs(): if self.applyToOddEven(ts): paramsTrimVol['input'] = oddEvenTmp[0] - paramsTrimVol['output'] = os.path.join(extraPrefix, tsId + EXT_MRC_ODD_NAME) + paramsTrimVol['output'] = self.getExtraOutFile(tsId, suffix=ODD, ext=MRCS_EXT) Plugin.runImod(self, 'trimvol', argsTrimvol % paramsTrimVol) paramsTrimVol['input'] = oddEvenTmp[1] - paramsTrimVol['output'] = os.path.join(extraPrefix, tsId + EXT_MRC_EVEN_NAME) + paramsTrimVol['output'] = self.getExtraOutFile(tsId, suffix=EVEN, ext=MRCS_EXT) Plugin.runImod(self, 'trimvol', argsTrimvol % paramsTrimVol) - def createOutputStep(self, tsObjId): - ts = self.inputSetOfTiltSeries.get()[tsObjId] - tsId = ts.getTsId() - firstItem = ts.getFirstItem() - - extraPrefix = self._getExtraPath(tsId) - - tomoLocation = os.path.join(extraPrefix, firstItem.parseFileName(extension=".mrc")) + def createOutputStep(self, tsId): + ts = self.tsDict[tsId] + tomoLocation = self.getExtraOutFile(tsId) if os.path.exists(tomoLocation): output = self.getOutputSetOfTomograms(self.inputSetOfTiltSeries.get()) @@ -304,8 +293,8 @@ def createOutputStep(self, tsObjId): newTomogram.setLocation(tomoLocation) if self.applyToOddEven(ts): - halfMapsList = [os.path.join(extraPrefix, tsId + EXT_MRC_ODD_NAME), - os.path.join(extraPrefix, tsId + EXT_MRC_EVEN_NAME)] + halfMapsList = [self.getExtraOutFile(tsId, suffix=ODD, ext=MRCS_EXT), + self.getExtraOutFile(tsId, suffix=EVEN, ext=MRCS_EXT)] newTomogram.setHalfMaps(halfMapsList) newTomogram.setTsId(tsId) @@ -325,6 +314,10 @@ def createOutputStep(self, tsObjId): output.write() self._store() + def createOutputFailedStep(self, tsId): + ts = self.tsDict[tsId] + super().createOutputFailedSet(ts) + def closeOutputSetsStep(self): for _, output in self.iterOutputAttributes(): output.setStreamState(Set.STREAM_CLOSED) diff --git a/imod/protocols/protocol_tsNormalization.py b/imod/protocols/protocol_tsPreprocess.py similarity index 91% rename from imod/protocols/protocol_tsNormalization.py rename to imod/protocols/protocol_tsPreprocess.py index cd390a9d..3474d173 100644 --- a/imod/protocols/protocol_tsNormalization.py +++ b/imod/protocols/protocol_tsPreprocess.py @@ -32,11 +32,12 @@ import tomo.objects as tomoObj from .. import Plugin -from .protocol_base import ProtImodBase, EXT_MRCS_TS_EVEN_NAME, EXT_MRCS_TS_ODD_NAME, OUTPUT_TILTSERIES_NAME +from .protocol_base import ProtImodBase, EXT_MRCS_TS_EVEN_NAME, EXT_MRCS_TS_ODD_NAME, OUTPUT_TILTSERIES_NAME, XF_EXT, \ + ODD, EVEN, MRCS_EXT from ..utils import formatTransformFile -class ProtImodTSNormalization(ProtImodBase): +class ProtImodTsPreprocess(ProtImodBase): """ Normalize input tilt-series and change its storing formatting. More info: @@ -45,7 +46,7 @@ class ProtImodTSNormalization(ProtImodBase): _label = 'Tilt-series preprocess' _devStatus = BETA - _possibleOutputs = {OUTPUT_TILTSERIES_NAME:tomoObj.SetOfTiltSeries} + _possibleOutputs = {OUTPUT_TILTSERIES_NAME: tomoObj.SetOfTiltSeries} # -------------------------- DEFINE param functions ----------------------- def _defineParams(self, form): @@ -200,42 +201,42 @@ def _defineParams(self, form): # -------------------------- INSERT steps functions ----------------------- def _insertAllSteps(self): - self._failedTs = [] - - for ts in self.inputSetOfTiltSeries.get(): - self._insertFunctionStep(self.convertInputStep, ts.getObjId()) - self._insertFunctionStep(self.generateOutputStackStep, ts.getObjId()) - self._insertFunctionStep(self.createOutputFailedSet, ts.getObjId()) + self._initialize() + for tsId in self.tsDict.keys(): + self._insertFunctionStep(self.convertInputStep, tsId) + self._insertFunctionStep(self.generateOutputStackStep, tsId) + self._insertFunctionStep(self.createOutputFailedStep, tsId) self._insertFunctionStep(self.closeOutputSetsStep) # --------------------------- STEPS functions ----------------------------- - def convertInputStep(self, tsObjId, **kwargs): + def _initialize(self): + self._failedTs = [] + self.tsDict = {ts.getTsId(): ts.clone(ignoreAttrs=[]) for ts in self.inputSetOfTiltSeries.get()} + + def convertInputStep(self, tsId, **kwargs): oddEvenFlag = self.applyToOddEven(self.inputSetOfTiltSeries.get()) # Interpolation will be done in the generateOutputStep - super().convertInputStep(tsObjId, imodInterpolation=None, - generateAngleFile=False, oddEven=oddEvenFlag) + super().convertInputStep(tsId, + imodInterpolation=None, + generateAngleFile=False, + oddEven=oddEvenFlag) @ProtImodBase.tryExceptDecorator def generateOutputStackStep(self, tsObjId): ts = self.inputSetOfTiltSeries.get()[tsObjId] tsId = ts.getTsId() - - extraPrefix = self._getExtraPath(tsId) - tmpPrefix = self._getTmpPath(tsId) firstItem = ts.getFirstItem() - xfFile = None if self.applyAlignment.get() and ts.hasAlignment(): - xfFile = os.path.join(tmpPrefix, firstItem.parseFileName(extension=".xf")) + xfFile = self.getExtraOutFile(tsId, ext=XF_EXT) formatTransformFile(ts, xfFile) binning = self.binning.get() argsNewstack, paramsNewstack = self.getBasicNewstackParams(ts, - os.path.join(extraPrefix, firstItem.parseFileName()), - inputTsFileName=os.path.join(tmpPrefix, - firstItem.parseFileName()), + self.getExtraOutFile(tsId), + inputTsFileName=self.getTmpOutFile(tsId), xfFile=xfFile, firstItem=firstItem, binning=binning, @@ -275,10 +276,10 @@ def generateOutputStackStep(self, tsObjId): oddFn = firstItem.getOdd().split('@')[1] evenFn = firstItem.getEven().split('@')[1] paramsNewstack['input'] = oddFn - paramsNewstack['output'] = os.path.join(extraPrefix, tsId + EXT_MRCS_TS_ODD_NAME) + paramsNewstack['output'] = self.getExtraOutFile(tsId, suffix=ODD, ext=MRCS_EXT) Plugin.runImod(self, 'newstack', argsNewstack % paramsNewstack) paramsNewstack['input'] = evenFn - paramsNewstack['output'] = os.path.join(extraPrefix, tsId + EXT_MRCS_TS_EVEN_NAME) + paramsNewstack['output'] = self.getExtraOutFile(tsId, suffix=EVEN, ext=MRCS_EXT) Plugin.runImod(self, 'newstack', argsNewstack % paramsNewstack) output = self.getOutputSetOfTiltSeries(self.inputSetOfTiltSeries.get(), self.binning.get()) @@ -304,14 +305,14 @@ def generateOutputStackStep(self, tsObjId): newTi.setAcquisition(tiltImage.getAcquisition()) if self.applyToOddEven(ts): - locationOdd = index + 1, (os.path.join(extraPrefix, tsId + EXT_MRCS_TS_ODD_NAME)) - locationEven = index + 1, (os.path.join(extraPrefix, tsId + EXT_MRCS_TS_EVEN_NAME)) + locationOdd = index + 1, self.getExtraOutFile(tsId, suffix=ODD, ext=MRCS_EXT) + locationEven = index + 1, self.getExtraOutFile(tsId, suffix=EVEN, ext=MRCS_EXT) newTi.setOddEven([ih.locationToXmipp(locationOdd), ih.locationToXmipp(locationEven)]) else: newTi.setOddEven([]) - newTi.setLocation(index + 1, - (os.path.join(extraPrefix, tiltImage.parseFileName()))) + newTi.setLocation(index + 1, self.getExtraOutFile(tsId)) + if binning > 1: newTi.setSamplingRate(tiltImage.getSamplingRate() * binning) newTs.append(newTi) @@ -325,6 +326,10 @@ def generateOutputStackStep(self, tsObjId): output.write() self._store() + def createOutputFailedStep(self, tsId): + ts = self.tsDict[tsId] + super().createOutputFailedSet(ts) + def closeOutputSetsStep(self): for _, output in self.iterOutputAttributes(): output.setStreamState(Set.STREAM_CLOSED) diff --git a/imod/protocols/protocol_xCorrPrealignment.py b/imod/protocols/protocol_xCorrPrealignment.py index 5c29b065..9c212fff 100644 --- a/imod/protocols/protocol_xCorrPrealignment.py +++ b/imod/protocols/protocol_xCorrPrealignment.py @@ -35,7 +35,7 @@ import tomo.objects as tomoObj from .. import Plugin, utils -from .protocol_base import ProtImodBase +from .protocol_base import ProtImodBase, TLT_EXT class ProtImodXcorrPrealignment(ProtImodBase): @@ -211,33 +211,28 @@ def _defineParams(self, form): # -------------------------- INSERT steps functions ----------------------- def _insertAllSteps(self): - for ts in self.inputSetOfTiltSeries.get(): - self._insertFunctionStep(self.convertInputStep, ts.getObjId()) - self._insertFunctionStep(self.computeXcorrStep, ts.getObjId()) - self._insertFunctionStep(self.generateOutputStackStep, - ts.getObjId()) + self._initialize() + for tsId in self.tsDict.keys(): + self._insertFunctionStep(self.convertInputStep, tsId) + self._insertFunctionStep(self.computeXcorrStep, tsId) + self._insertFunctionStep(self.generateOutputStackStep,tsId) if self.computeAlignment.get() == 0: - self._insertFunctionStep(self.computeInterpolatedStackStep, - ts.getObjId()) + self._insertFunctionStep(self.computeInterpolatedStackStep, tsId) self._insertFunctionStep(self.closeOutputSetsStep) # --------------------------- STEPS functions ----------------------------- - def computeXcorrStep(self, tsObjId): - """Compute transformation matrix for each tilt series""" - ts = self.inputSetOfTiltSeries.get()[tsObjId] - tsId = ts.getTsId() - extraPrefix = self._getExtraPath(tsId) - tmpPrefix = self._getTmpPath(tsId) + def _initialize(self): + self.tsDict = {ts.getTsId(): ts.clone(ignoreAttrs=[]) for ts in self.inputSetOfTiltSeries.get()} + def computeXcorrStep(self, tsId): + """Compute transformation matrix for each tilt series""" + ts = self.tsDict[tsId] tiltAxisAngle = self.getTiltAxisOrientation(ts) paramsXcorr = { - 'input': os.path.join(tmpPrefix, - ts.getFirstItem().parseFileName()), - 'output': os.path.join(extraPrefix, - ts.getFirstItem().parseFileName(extension=".prexf")), - 'tiltfile': os.path.join(tmpPrefix, - ts.getFirstItem().parseFileName(extension=".tlt")), + 'input': self.getTmpOutFile(tsId), + 'output': self.getExtraOutFile(tsId, ext='prexf'), + 'tiltfile': self.getExtraOutFile(tsId, ext=TLT_EXT), 'rotationAngle': tiltAxisAngle, 'filterSigma1': self.filterSigma1.get(), 'filterSigma2': self.filterSigma2.get(), @@ -284,39 +279,22 @@ def computeXcorrStep(self, tsObjId): Plugin.runImod(self, 'tiltxcorr', argsXcorr % paramsXcorr) paramsXftoxg = { - 'input': os.path.join(extraPrefix, - ts.getFirstItem().parseFileName(extension=".prexf")), - 'goutput': os.path.join(extraPrefix, - ts.getFirstItem().parseFileName(extension=".prexg")), + 'input': self.getExtraOutFile(tsId, ext='prexf'), + 'goutput': self.getExtraOutFile(tsId, ext='prexg'), } argsXftoxg = "-input %(input)s " \ "-NumberToFit 0 " \ "-goutput %(goutput)s " Plugin.runImod(self, 'xftoxg', argsXftoxg % paramsXftoxg) - def getTiltAxisOrientation(self, ts): - if self.tiltAxisAngle.get(): - return self.tiltAxisAngle.get() - else: - return ts.getAcquisition().getTiltAxisAngle() - - def generateOutputStackStep(self, tsObjId): + def generateOutputStackStep(self, tsId): """ Generate tilt-serie with the associated transform matrix """ - ts = self.inputSetOfTiltSeries.get()[tsObjId] - tsId = ts.getTsId() - - extraPrefix = self._getExtraPath(tsId) - + ts = self.tsDict[tsId] output = self.getOutputSetOfTiltSeries(self.inputSetOfTiltSeries.get()) - - alignmentMatrix = utils.formatTransformationMatrix( - os.path.join(extraPrefix, - ts.getFirstItem().parseFileName(extension=".prexg"))) - + alignmentMatrix = utils.formatTransformationMatrix(self.getExtraOutFile(tsId, ext='prexg')) newTs = tomoObj.TiltSeries(tsId=tsId) newTs.copyInfo(ts) newTs.getAcquisition().setTiltAxisAngle(self.getTiltAxisOrientation(ts)) - output.append(newTs) for index, tiltImage in enumerate(ts): @@ -352,20 +330,14 @@ def generateOutputStackStep(self, tsObjId): self._store() - def computeInterpolatedStackStep(self, tsObjId): + def computeInterpolatedStackStep(self, tsId): output = self.getOutputInterpolatedSetOfTiltSeries(self.inputSetOfTiltSeries.get()) - - ts = self.inputSetOfTiltSeries.get()[tsObjId] - tsId = ts.getTsId() - - extraPrefix = self._getExtraPath(tsId) - tmpPrefix = self._getTmpPath(tsId) + ts = self.tsDict[tsId] paramsAlignment = { - 'input': os.path.join(tmpPrefix, ts.getFirstItem().parseFileName()), - 'output': os.path.join(extraPrefix, ts.getFirstItem().parseFileName()), - 'xform': os.path.join(extraPrefix, - ts.getFirstItem().parseFileName(extension=".prexg")), + 'input': self.getTmpOutFile(tsId), + 'output': self.getExtraOutFile(tsId), + 'xform': self.getExtraOutFile(tsId, ext='prexg'), 'bin': self.binning.get(), 'imagebinned': 1.0 } @@ -392,9 +364,7 @@ def computeInterpolatedStackStep(self, tsObjId): for index, tiltImage in enumerate(ts): newTi = tomoObj.TiltImage() newTi.copyInfo(tiltImage, copyId=True) - newTi.setLocation(index + 1, - (os.path.join(extraPrefix, - tiltImage.parseFileName()))) + newTi.setLocation(index + 1, self.getExtraOutFile(tsId)) if self.binning > 1: newTi.setSamplingRate(tiltImage.getSamplingRate() * self.binning.get()) newTs.append(newTi) @@ -418,6 +388,13 @@ def closeOutputSetsStep(self): self._store() + # --------------------------- UTILS functions ------------------------------ + def getTiltAxisOrientation(self, ts): + if self.tiltAxisAngle.get(): + return self.tiltAxisAngle.get() + else: + return ts.getAcquisition().getTiltAxisAngle() + # --------------------------- INFO functions ------------------------------ def _summary(self): summary = [] diff --git a/imod/protocols/protocol_xRaysEraser.py b/imod/protocols/protocol_xRaysEraser.py index d8b032e6..633cb4f1 100644 --- a/imod/protocols/protocol_xRaysEraser.py +++ b/imod/protocols/protocol_xRaysEraser.py @@ -33,7 +33,8 @@ import tomo.objects as tomoObj from .. import Plugin -from .protocol_base import ProtImodBase, OUTPUT_TILTSERIES_NAME, EXT_MRCS_TS_ODD_NAME, EXT_MRCS_TS_EVEN_NAME +from .protocol_base import ProtImodBase, OUTPUT_TILTSERIES_NAME, EXT_MRCS_TS_ODD_NAME, EXT_MRCS_TS_EVEN_NAME, ODD, \ + MRCS_EXT, EVEN class ProtImodXraysEraser(ProtImodBase): @@ -111,29 +112,30 @@ def _defineParams(self, form): # -------------------------- INSERT steps functions ----------------------- def _insertAllSteps(self): - for ts in self.inputSetOfTiltSeries.get(): - self._insertFunctionStep(self.convertInputStep, ts.getObjId()) - self._insertFunctionStep(self.eraseXraysStep, ts.getObjId()) - self._insertFunctionStep(self.createOutputStep, ts.getObjId()) + self._initialize() + for tsId in self.tsDict.keys(): + self._insertFunctionStep(self.convertInputStep, tsId) + self._insertFunctionStep(self.eraseXraysStep, tsId) + self._insertFunctionStep(self.createOutputStep, tsId) self._insertFunctionStep(self.closeOutputStep) - def convertInputStep(self, tsObjId, **kwargs): - oddEvenFlag = self.applyToOddEven(self.inputSetOfTiltSeries.get()) - super().convertInputStep(tsObjId, imodInterpolation=None, - generateAngleFile=False, oddEven=oddEvenFlag) - - def eraseXraysStep(self, tsObjId): - ts = self.inputSetOfTiltSeries.get()[tsObjId] + def _initialize(self): + self.tsDict = {ts.getTsId(): ts.clone(ignoreAttrs=[]) for ts in self.inputSetOfTiltSeries.get()} - tsId = ts.getTsId() - extraPrefix = self._getExtraPath(tsId) - tmpPrefix = self._getTmpPath(tsId) + def convertInputStep(self, tsId, **kwargs): + oddEvenFlag = self.applyToOddEven(self.inputSetOfTiltSeries.get()) + super().convertInputStep(tsId, + imodInterpolation=None, + generateAngleFile=False, + oddEven=oddEvenFlag) + def eraseXraysStep(self, tsId): + ts = self.tsDict[tsId] firstItem = ts.getFirstItem() paramsCcderaser = { - 'input': os.path.join(tmpPrefix, firstItem.parseFileName()), - 'output': os.path.join(extraPrefix, firstItem.parseFileName()), + 'input': self.getTmpOutFile(tsId), + 'output': self.getExtraOutFile(tsId), 'findPeaks': 1, 'peakCriterion': self.peakCriterion.get(), 'diffCriterion': self.diffCriterion.get(), @@ -146,9 +148,7 @@ def eraseXraysStep(self, tsObjId): 'annulusWidth': 2.0, 'xyScanSize': 100, 'edgeExclusionWidth': 4, - 'pointModel': os.path.join(extraPrefix, - firstItem.parseFileName(suffix="_fid", - extension=".mod")), + 'pointModel': self.getExtraOutFile(tsId, suffix="fid", ext="mod"), 'borderSize': 2, 'polynomialOrder': 2, } @@ -176,19 +176,16 @@ def eraseXraysStep(self, tsObjId): oddFn = firstItem.getOdd().split('@')[1] evenFn = firstItem.getEven().split('@')[1] paramsCcderaser['input'] = oddFn - paramsCcderaser['output'] = os.path.join(extraPrefix, tsId + EXT_MRCS_TS_ODD_NAME) + paramsCcderaser['output'] = self.getExtraOutFile(tsId, suffix=ODD, ext=MRCS_EXT) Plugin.runImod(self, 'ccderaser', argsCcderaser % paramsCcderaser) paramsCcderaser['input'] = evenFn - paramsCcderaser['output'] = os.path.join(extraPrefix, tsId + EXT_MRCS_TS_EVEN_NAME) + paramsCcderaser['output'] = self.getExtraOutFile(tsId, suffix=EVEN, ext=MRCS_EXT) Plugin.runImod(self, 'ccderaser', argsCcderaser % paramsCcderaser) - def createOutputStep(self, tsObjId): + def createOutputStep(self, tsId): output = self.getOutputSetOfTiltSeries(self.inputSetOfTiltSeries.get()) - ts = self.inputSetOfTiltSeries.get()[tsObjId] - tsId = ts.getTsId() - extraPrefix = self._getExtraPath(tsId) - + ts = self.tsDict[tsId] newTs = tomoObj.TiltSeries(tsId=tsId) newTs.copyInfo(ts) output.append(newTs) @@ -199,12 +196,11 @@ def createOutputStep(self, tsObjId): newTi = tomoObj.TiltImage() newTi.copyInfo(tiltImage, copyId=True, copyTM=True) newTi.setAcquisition(tiltImage.getAcquisition()) - newTi.setLocation(index + 1, - (os.path.join(extraPrefix, - tiltImage.parseFileName()))) + newTi.setLocation(index + 1, self.getExtraOutFile(tsId)) + if self.applyToOddEven(ts): - locationOdd = index + 1, (os.path.join(extraPrefix, tsId + EXT_MRCS_TS_ODD_NAME)) - locationEven = index + 1, (os.path.join(extraPrefix, tsId + EXT_MRCS_TS_EVEN_NAME)) + locationOdd = index + 1, self.getExtraOutFile(tsId, suffix=ODD, ext=MRCS_EXT) + locationEven = index + 1, self.getExtraOutFile(tsId, suffix=EVEN, ext=MRCS_EXT) newTi.setOddEven([ih.locationToXmipp(locationOdd), ih.locationToXmipp(locationEven)]) else: newTi.setOddEven([]) diff --git a/imod/tests/test_protocols_imod.py b/imod/tests/test_protocols_imod.py index ad94f3e5..fcb6109d 100644 --- a/imod/tests/test_protocols_imod.py +++ b/imod/tests/test_protocols_imod.py @@ -108,7 +108,7 @@ def _runTSNormalization(cls, inputSoTS, binning, floatDensities, modeToOutput, scaleRangeToggle, scaleRangeMax, scaleRangeMin, meanSdToggle, scaleMean, scaleSd, scaleMax, scaleMin): - cls.protTSNormalization = cls.newProtocol(ProtImodTSNormalization, + cls.protTSNormalization = cls.newProtocol(ProtImodTsPreprocess, inputSetOfTiltSeries=inputSoTS, binning=binning, floatDensities=floatDensities, From 49b2c583740a837e2ec5eab81fafb082e489cac5 Mon Sep 17 00:00:00 2001 From: JorMaister Date: Thu, 11 Apr 2024 10:53:18 +0200 Subject: [PATCH 17/49] Update tests, not finished --- imod/tests/test_protocols_imod.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/imod/tests/test_protocols_imod.py b/imod/tests/test_protocols_imod.py index fcb6109d..a3140c6b 100644 --- a/imod/tests/test_protocols_imod.py +++ b/imod/tests/test_protocols_imod.py @@ -33,6 +33,9 @@ class TestImodBase(BaseTest): + atsId = 'a' + btsId = 'b' + @classmethod def setUpClass(cls): setupTestProject(cls) @@ -308,7 +311,7 @@ def setUpClass(cls): cls.inputTMFolder = os.path.split(cls.inputDataSet.getFile('tm1'))[0] - cls.excludeViewsOutputSizes = {'a': 57, 'b': 56} + cls.excludeViewsOutputSizes = {cls.atsId: 57, cls.btsId: 56} cls.binningTsNormalization = 2 @@ -594,7 +597,7 @@ def test_applyTransformationMatrixOutputInterpolatedTS(self): tsId = output.getFirstItem().getTsId() outputLocation = os.path.join(self.protApplyTransformationMatrix._getExtraPath(tsId), - "BB" + tsId + ".st") + tsId + ".st") self.assertTrue(os.path.exists(outputLocation)) From 58aad1c4707b0172e940dd72654999bb2454c72e Mon Sep 17 00:00:00 2001 From: JorMaister Date: Thu, 11 Apr 2024 13:19:20 +0200 Subject: [PATCH 18/49] Refactor finished, tests passing. Only the defocus file generateion cases still pending --- imod/protocols/__init__.py | 2 +- .../protocols/deprecated_20240411/__init__.py | 1 + .../protocol_goldBeadEraser.py | 4 +- .../protocol_applyTransformationMatrix.py | 5 +- imod/protocols/protocol_base.py | 8 +-- imod/protocols/protocol_doseFilter.py | 6 +-- imod/protocols/protocol_goldBeadPicker3d.py | 54 ++++++++----------- imod/protocols/protocol_importSetOfTM.py | 29 +++++----- ...lization.py => protocol_tomoPreprocess.py} | 48 ++++++++--------- imod/protocols/protocol_tomoProjection.py | 49 +++++++---------- imod/protocols/protocol_tomoReconstruction.py | 8 +-- imod/protocols/protocol_tsPreprocess.py | 5 +- imod/tests/test_protocols_imod.py | 34 ++++++------ 13 files changed, 107 insertions(+), 146 deletions(-) create mode 100644 imod/protocols/deprecated_20240411/__init__.py rename imod/protocols/{ => deprecated_20240411}/protocol_goldBeadEraser.py (98%) rename imod/protocols/{protocol_tomoNormalization.py => protocol_tomoPreprocess.py} (90%) diff --git a/imod/protocols/__init__.py b/imod/protocols/__init__.py index cf0a04fe..d8869b18 100644 --- a/imod/protocols/__init__.py +++ b/imod/protocols/__init__.py @@ -40,7 +40,7 @@ from .protocol_fiducialAlignment import ProtImodFiducialAlignment from .protocol_fiducialModel import ProtImodFiducialModel from .protocol_goldBeadPicker3d import ProtImodGoldBeadPicker3d -from .protocol_tomoNormalization import ProtImodTomoNormalization +from .protocol_tomoPreprocess import ProtImodTomoPreProcess from .protocol_tomoProjection import ProtImodTomoProjection from .protocol_tomoReconstruction import ProtImodTomoReconstruction from .protocol_tsPreprocess import ProtImodTsPreprocess diff --git a/imod/protocols/deprecated_20240411/__init__.py b/imod/protocols/deprecated_20240411/__init__.py new file mode 100644 index 00000000..47ea1229 --- /dev/null +++ b/imod/protocols/deprecated_20240411/__init__.py @@ -0,0 +1 @@ +# Gold bead eraser: it does not seem to work fine. We may back to it in the future... \ No newline at end of file diff --git a/imod/protocols/protocol_goldBeadEraser.py b/imod/protocols/deprecated_20240411/protocol_goldBeadEraser.py similarity index 98% rename from imod/protocols/protocol_goldBeadEraser.py rename to imod/protocols/deprecated_20240411/protocol_goldBeadEraser.py index e8546a16..19e5af72 100644 --- a/imod/protocols/protocol_goldBeadEraser.py +++ b/imod/protocols/deprecated_20240411/protocol_goldBeadEraser.py @@ -34,8 +34,8 @@ from tomo.protocols import ProtTomoBase import tomo.objects as tomoObj -from .. import Plugin, utils -from .protocol_base import OUTPUT_TILTSERIES_NAME +from imod import Plugin, utils +from imod.protocols.protocol_base import OUTPUT_TILTSERIES_NAME class ProtImodGoldBeadEraser(EMProtocol, ProtTomoBase): diff --git a/imod/protocols/protocol_applyTransformationMatrix.py b/imod/protocols/protocol_applyTransformationMatrix.py index 5ec2e2c9..20ac2c66 100644 --- a/imod/protocols/protocol_applyTransformationMatrix.py +++ b/imod/protocols/protocol_applyTransformationMatrix.py @@ -25,16 +25,13 @@ # ***************************************************************************** import os - from pyworkflow import BETA import pyworkflow.protocol.params as params -import pyworkflow.utils.path as path from pyworkflow.object import Set from pwem.emlib.image import ImageHandler from tomo.objects import TiltSeries, TiltImage - from .. import Plugin, utils -from .protocol_base import ProtImodBase, EXT_MRCS_TS_EVEN_NAME, EXT_MRCS_TS_ODD_NAME, XF_EXT, ODD, EVEN, MRCS_EXT +from .protocol_base import ProtImodBase,XF_EXT, ODD, EVEN, MRCS_EXT class ProtImodApplyTransformationMatrix(ProtImodBase): diff --git a/imod/protocols/protocol_base.py b/imod/protocols/protocol_base.py index 195d428f..62a2ce79 100644 --- a/imod/protocols/protocol_base.py +++ b/imod/protocols/protocol_base.py @@ -54,6 +54,7 @@ EVEN = 'even' ODD = 'odd' MRCS_EXT = 'mrcs' +MRC_EXT = 'mrc' XF_EXT = 'xf' TLT_EXT = 'tlt' DEFOCUS_EXT = 'defocus' @@ -68,6 +69,7 @@ def __init__(self, **args): # Possible outputs (synchronize these names with the constants) self.tsDict = None + self.tomoDict = None self.binning = 1 self._failedTs = [] self.TiltSeriesCoordinates = None @@ -122,13 +124,13 @@ def genTsPaths(self, tsId): path.makePath(*[self._getExtraPath(tsId), self._getTmpPath(tsId)]) @staticmethod - def getOutTsFileName(tsId, suffix=None, ext='mrc'): + def getOutTsFileName(tsId, suffix=None, ext='st'): return f'{tsId}_{suffix}.{ext}' if suffix else f'{tsId}.{ext}' - def getTmpOutFile(self, tsId, suffix=None, ext='mrc'): + def getTmpOutFile(self, tsId, suffix=None, ext='st'): return self._getTmpPath(tsId, self.getOutTsFileName(tsId, suffix=suffix, ext=ext)) - def getExtraOutFile(self, tsId, suffix=None, ext='mrc'): + def getExtraOutFile(self, tsId, suffix=None, ext='st'): return self._getExtraPath(tsId, self.getOutTsFileName(tsId, suffix=suffix, ext=ext)) def convertInputStep(self, tsObjId, generateAngleFile=True, imodInterpolation=True, doSwap=False, diff --git a/imod/protocols/protocol_doseFilter.py b/imod/protocols/protocol_doseFilter.py index 065c67f1..392eeffb 100644 --- a/imod/protocols/protocol_doseFilter.py +++ b/imod/protocols/protocol_doseFilter.py @@ -23,18 +23,14 @@ # * e-mail address 'scipion@cnb.csic.es' # * # ***************************************************************************** - -import os - from pyworkflow import BETA from pyworkflow.object import Set import pyworkflow.protocol.params as params -import pyworkflow.utils.path as path import tomo.objects as tomoObj from pwem.emlib.image import ImageHandler from .. import Plugin, utils -from .protocol_base import ProtImodBase, EXT_MRCS_TS_EVEN_NAME, EXT_MRCS_TS_ODD_NAME, ODD, MRCS_EXT, EVEN +from .protocol_base import ProtImodBase, ODD, MRCS_EXT, EVEN SCIPION_IMPORT = 0 FIXED_DOSE = 1 diff --git a/imod/protocols/protocol_goldBeadPicker3d.py b/imod/protocols/protocol_goldBeadPicker3d.py index 8abc6119..a8de4636 100644 --- a/imod/protocols/protocol_goldBeadPicker3d.py +++ b/imod/protocols/protocol_goldBeadPicker3d.py @@ -107,39 +107,39 @@ def _defineParams(self, form): # -------------------------- INSERT steps functions ----------------------- def _insertAllSteps(self): - self.defineExecutionPararell() - allOutputId = [] - - for ts in self.inputSetOfTomograms.get(): + self._initialize() + for tsId in self.tomoDict.keys(): pickId = self._insertFunctionStep(self.pickGoldBeadsStep, - ts.getObjId(), + tsId, prerequisites=[]) convertId = self._insertFunctionStep(self.convertModelToCoordinatesStep, - ts.getObjId(), + tsId, prerequisites=[pickId]) outputID = self._insertFunctionStep(self.createOutputStep, - ts.getObjId(), + tsId, prerequisites=[convertId]) allOutputId.append(outputID) - self._insertFunctionStep('closeOutputSetStep', + self._insertFunctionStep(self.closeOutputSetStep, prerequisites=allOutputId) # --------------------------- STEPS functions ----------------------------- - def pickGoldBeadsStep(self, tsObjId): - tomo = self.inputSetOfTomograms.get()[tsObjId] - fileName = removeBaseExt(tomo.getFileName()) - extraPrefix = self._getExtraPath(fileName) - path.makePath(extraPrefix) + def _initialize(self): + self.defineExecutionPararell() + self.tomoDict = {tomo.getTsId(): tomo.clone() for tomo in self.inputSetOfTomograms.get()} + + def pickGoldBeadsStep(self, tsId): + self.genTsPaths(tsId) + tomo = self.tomoDict[tsId] """ Run findbeads3d IMOD program """ paramsFindbeads3d = { 'inputFile': tomo.getFileName(), - 'outputFile': os.path.join(extraPrefix, "%s.mod" % fileName), + 'outputFile': self.getExtraOutFile(tsId, ext='mod'), 'beadSize': self.beadDiameter.get(), 'minRelativeStrength': self.minRelativeStrength.get(), 'minSpacing': self.minSpacing.get(), @@ -157,17 +157,11 @@ def pickGoldBeadsStep(self, tsObjId): Plugin.runImod(self, 'findbeads3d', argsFindbeads3d % paramsFindbeads3d) - def convertModelToCoordinatesStep(self, tsObjId): - tomo = self.inputSetOfTomograms.get()[tsObjId] - location = tomo.getFileName() - fileName, _ = os.path.splitext(location) - - extraPrefix = self._getExtraPath(os.path.basename(fileName)) - + def convertModelToCoordinatesStep(self, tsId): """ Run model2point IMOD program """ paramsModel2Point = { - 'inputFile': os.path.join(extraPrefix, "%s.mod" % os.path.basename(fileName)), - 'outputFile': os.path.join(extraPrefix, "%s.xyz" % os.path.basename(fileName)), + 'inputFile': self.getExtraOutFile(tsId, ext='mod'), + 'outputFile': self.getExtraOutFile(tsId, ext='xyz'), } argsModel2Point = "-InputFile %(inputFile)s " \ @@ -175,20 +169,14 @@ def convertModelToCoordinatesStep(self, tsObjId): Plugin.runImod(self, 'model2point', argsModel2Point % paramsModel2Point) - def createOutputStep(self, tsObjId): - tomo = self.inputSetOfTomograms.get()[tsObjId] - location = tomo.getFileName() - fileName, _ = os.path.splitext(location) - - extraPrefix = self._getExtraPath(os.path.basename(fileName)) + def createOutputStep(self, tsId): + tomo = self.tomoDict[tsId] """ Create the output set of coordinates 3D from gold beads detected """ output = self.getOutputSetOfCoordinates3Ds(self.inputSetOfTomograms.get(), self.inputSetOfTomograms.get()) - coordFilePath = os.path.join(extraPrefix, - "%s.xyz" % os.path.basename(fileName)) - + coordFilePath = self.getExtraOutFile(tsId, ext='xyz') coordList = utils.formatGoldBead3DCoordinatesList(coordFilePath) with self._lock: @@ -199,7 +187,7 @@ def createOutputStep(self, tsObjId): newCoord3D.setY(element[1], constants.BOTTOM_LEFT_CORNER) newCoord3D.setZ(element[2], constants.BOTTOM_LEFT_CORNER) - newCoord3D.setVolId(tsObjId) + # newCoord3D.setVolId(tsObjId) output.append(newCoord3D) output.update(newCoord3D) diff --git a/imod/protocols/protocol_importSetOfTM.py b/imod/protocols/protocol_importSetOfTM.py index d35f16d6..48fc2256 100644 --- a/imod/protocols/protocol_importSetOfTM.py +++ b/imod/protocols/protocol_importSetOfTM.py @@ -83,25 +83,23 @@ def _defineParams(self, form): # -------------------------- INSERT steps functions ----------------------- def _insertAllSteps(self): - self.matchBinningFactor = self.binningTM.get() / self.binningTS.get() - - for ts in self.inputSetOfTiltSeries.get(): - self._insertFunctionStep(self.generateTransformFileStep, - ts.getObjId()) - self._insertFunctionStep(self.assignTransformationMatricesStep, - ts.getObjId()) + self._initialize() + for tsId in self.tsDict.keys(): + self._insertFunctionStep(self.generateTransformFileStep, tsId) + self._insertFunctionStep(self.assignTransformationMatricesStep, tsId) self._insertFunctionStep(self.closeOutputSetsStep) # --------------------------- STEPS functions ----------------------------- - def generateTransformFileStep(self, tsObjId): - ts = self.inputSetOfTiltSeries.get()[tsObjId] - tsId = ts.getTsId() + def _initialize(self): + self.matchBinningFactor = self.binningTM.get() / self.binningTS.get() + self.tsDict = {ts.getTsId(): ts.clone(ignoreAttrs=[]) for ts in self.inputSetOfTiltSeries.get()} + def generateTransformFileStep(self, tsId): + self.genTsPaths(tsId) + ts = self.tsDict[tsId] tsFileName = ts.getFirstItem().parseFileName(extension='') - extraPrefix = self._getExtraPath(tsId) - path.makePath(extraPrefix) outputTransformFile = os.path.join(extraPrefix, ts.getFirstItem().parseFileName(extension=".xf")) @@ -147,12 +145,9 @@ def generateTransformFileStep(self, tsObjId): else: path.createLink(tmFilePath, outputTransformFile) - def assignTransformationMatricesStep(self, tsObjId): - ts = self.inputSetOfTiltSeries.get()[tsObjId] - tsId = ts.getTsId() - + def assignTransformationMatricesStep(self, tsId): + ts = self.tsDict[tsId] extraPrefix = self._getExtraPath(tsId) - outputTransformFile = os.path.join(extraPrefix, ts.getFirstItem().parseFileName(extension=".xf")) diff --git a/imod/protocols/protocol_tomoNormalization.py b/imod/protocols/protocol_tomoPreprocess.py similarity index 90% rename from imod/protocols/protocol_tomoNormalization.py rename to imod/protocols/protocol_tomoPreprocess.py index f1b0985f..9d402005 100644 --- a/imod/protocols/protocol_tomoNormalization.py +++ b/imod/protocols/protocol_tomoPreprocess.py @@ -33,10 +33,10 @@ from tomo.objects import Tomogram, SetOfTomograms from .. import Plugin -from .protocol_base import ProtImodBase, EXT_MRC_ODD_NAME, EXT_MRC_EVEN_NAME, OUTPUT_TOMOGRAMS_NAME +from .protocol_base import ProtImodBase, EXT_MRC_ODD_NAME, EXT_MRC_EVEN_NAME, OUTPUT_TOMOGRAMS_NAME, MRC_EXT, ODD, EVEN -class ProtImodTomoNormalization(ProtImodBase): +class ProtImodTomoPreProcess(ProtImodBase): """ Normalize input tomogram and change its storing formatting. More info: @@ -194,23 +194,20 @@ def _defineParams(self, form): # -------------------------- INSERT steps functions ----------------------- def _insertAllSteps(self): - for tomo in self.inputSetOfTomograms.get(): - self._insertFunctionStep(self.generateOutputStackStep, - tomo.getObjId()) + self._initialize() + for tsId in self.tomoDict.keys(): + self._insertFunctionStep(self.generateOutputStackStep, tsId) self._insertFunctionStep(self.closeOutputSetsStep) # --------------------------- STEPS functions ----------------------------- - def generateOutputStackStep(self, tsObjId): - tomo = self.inputSetOfTomograms.get()[tsObjId] - location = tomo.getFileName() - fileName = os.path.splitext(location)[0] + def _initialize(self): + self.tomoDict = {tomo.getTsId(): tomo.clone() for tomo in self.inputSetOfTomograms.get()} - extraPrefix = self._getExtraPath(os.path.basename(fileName)) - outputFileNameBase = os.path.basename(pwpath.replaceExt(location, "mrc")) - outputFile = os.path.join(extraPrefix, outputFileNameBase) - tmpPrefix = self._getTmpPath(os.path.basename(fileName)) - pwpath.makePath(extraPrefix) - pwpath.makePath(tmpPrefix) + def generateOutputStackStep(self, tsId): + self.genTsPaths(tsId) + tomo = self.tomoDict[tsId] + location = tomo.getFileName() + outputFile = self.getExtraOutFile(tsId, ext=MRC_EXT) runNewstack = False @@ -253,11 +250,11 @@ def generateOutputStackStep(self, tsObjId): if self.applyToOddEven(tomo): oddFn, evenFn = tomo.getHalfMaps().split(',') paramsNewstack['input'] = oddFn - oddEvenOutput[0] = os.path.join(extraPrefix, tomo.getTsId() + EXT_MRC_ODD_NAME) + oddEvenOutput[0] = self.getExtraOutFile(tsId, suffix=ODD, ext=MRC_EXT) paramsNewstack['output'] = oddEvenOutput[0] Plugin.runImod(self, 'newstack', argsNewstack % paramsNewstack) paramsNewstack['input'] = evenFn - oddEvenOutput[1] = os.path.join(extraPrefix, tomo.getTsId() + EXT_MRC_EVEN_NAME) + oddEvenOutput[1] = self.getExtraOutFile(tsId, suffix=EVEN, ext=MRC_EXT) paramsNewstack['output'] = oddEvenOutput[1] Plugin.runImod(self, 'newstack', argsNewstack % paramsNewstack) @@ -265,17 +262,16 @@ def generateOutputStackStep(self, tsObjId): if binning != 1: if runNewstack: - baseLoc = os.path.basename(outputFile) - tmpPath = os.path.join(tmpPrefix, baseLoc) - pwpath.moveFile(os.path.join(extraPrefix, baseLoc), tmpPath) + tmpPath = self.getTmpOutFile(tsId, ext=MRC_EXT) + pwpath.moveFile(outputFile, tmpPath) inputTomoPath = tmpPath if self.applyToOddEven(tomo): pwpath.moveFile(oddEvenOutput[0], tmpPath) pwpath.moveFile(oddEvenOutput[1], tmpPath) inputTomoPath = tmpPath - inputOdd, inputEven = (os.path.join(tmpPrefix, tomo.getTsId() + EXT_MRC_ODD_NAME), - os.path.join(tmpPrefix, tomo.getTsId() + EXT_MRC_EVEN_NAME)) + inputOdd, inputEven = (self.getTmpOutFile(tsId, suffix=ODD, ext=MRC_EXT), + self.getTmpOutFile(tsId, suffix=EVEN, ext=MRC_EXT)) else: inputTomoPath = location if self.applyToOddEven(tomo): @@ -297,10 +293,10 @@ def generateOutputStackStep(self, tsObjId): if self.applyToOddEven(tomo): paramsBinvol['input'] = inputOdd - paramsBinvol['output'] = os.path.join(extraPrefix, tomo.getTsId() + EXT_MRC_ODD_NAME) + paramsBinvol['output'] = self.getExtraOutFile(tsId, suffix=ODD, ext=MRC_EXT) Plugin.runImod(self, 'binvol', argsBinvol % paramsBinvol) paramsBinvol['input'] = inputEven - paramsBinvol['output'] = os.path.join(extraPrefix, tomo.getTsId() + EXT_MRC_EVEN_NAME) + paramsBinvol['output'] = self.getExtraOutFile(tsId, suffix=EVEN, ext=MRC_EXT) Plugin.runImod(self, 'binvol', argsBinvol % paramsBinvol) output = self.getOutputSetOfTomograms(self.inputSetOfTomograms.get(), @@ -329,8 +325,8 @@ def generateOutputStackStep(self, tsObjId): newTomogram.copyAttributes(tomo, '_origin') if self.applyToOddEven(tomo): - halfMapsList = [os.path.join(extraPrefix, tomo.getTsId() + EXT_MRC_ODD_NAME), - os.path.join(extraPrefix, tomo.getTsId() + EXT_MRC_EVEN_NAME)] + halfMapsList = [self.getExtraOutFile(tsId, suffix=ODD, ext=MRC_EXT), + self.getExtraOutFile(tsId, suffix=EVEN, ext=MRC_EXT)] newTomogram.setHalfMaps(halfMapsList) output.append(newTomogram) diff --git a/imod/protocols/protocol_tomoProjection.py b/imod/protocols/protocol_tomoProjection.py index e1df8f76..3b4c4386 100644 --- a/imod/protocols/protocol_tomoProjection.py +++ b/imod/protocols/protocol_tomoProjection.py @@ -35,7 +35,7 @@ import tomo.objects as tomoObj from .. import Plugin -from .protocol_base import ProtImodBase +from .protocol_base import ProtImodBase, MRC_EXT class ProtImodTomoProjection(ProtImodBase): @@ -91,26 +91,23 @@ def _defineParams(self, form): # -------------------------- INSERT steps functions ----------------------- def _insertAllSteps(self): - for tomo in self.inputSetOfTomograms.get(): - self._insertFunctionStep(self.projectTomogram, tomo.getObjId()) - self._insertFunctionStep(self.generateOutputStackStep, tomo.getObjId()) + self._initialize() + for tsId in self.tomoDict.keys(): + self._insertFunctionStep(self.projectTomogram, tsId) + self._insertFunctionStep(self.generateOutputStackStep, tsId) self._insertFunctionStep(self.closeOutputSetsStep) # --------------------------- STEPS functions ----------------------------- - def projectTomogram(self, tomoObjId): - tomo = self.inputSetOfTomograms.get()[tomoObjId] + def _initialize(self): + self.tomoDict = {tomo.getTsId(): tomo.clone() for tomo in self.inputSetOfTomograms.get()} - tomoId = os.path.splitext(os.path.basename(tomo.getFileName()))[0] - - extraPrefix = self._getExtraPath(tomoId) - tmpPrefix = self._getTmpPath(tomoId) - path.makePath(tmpPrefix) - path.makePath(extraPrefix) + def projectTomogram(self, tsId): + self.genTsPaths(tsId) + tomo = self.tomoDict[tsId] paramsXYZproj = { 'input': tomo.getFileName(), - 'output': os.path.join(extraPrefix, - os.path.basename(tomo.getFileName())), + 'output': self.getExtraOutFile(tsId, ext=MRC_EXT), 'axis': self.getRotationAxis(), 'angles': str(self.minAngle.get()) + ',' + str(self.maxAngle.get()) + ',' + @@ -124,35 +121,25 @@ def projectTomogram(self, tomoObjId): Plugin.runImod(self, 'xyzproj', argsXYZproj % paramsXYZproj) - def generateOutputStackStep(self, tomoObjId): - tomo = self.inputSetOfTomograms.get()[tomoObjId] - - tomoId = os.path.splitext(os.path.basename(tomo.getFileName()))[0] - - extraPrefix = self._getExtraPath(tomoId) - + def generateOutputStackStep(self, tsId): + tomo = self.tomoDict[tsId] output = self.getOutputSetOfTiltSeries(self.inputSetOfTomograms.get()) - - newTs = tomoObj.TiltSeries(tsId=tomoId) - + newTs = tomoObj.TiltSeries(tsId=tsId) newTs.setTsId(tomo.getTsId()) newTs.setAcquisition(tomo.getAcquisition()) - newTs.setTsId(tomoId) # Add origin to output tilt-series output.append(newTs) - tiltAngleList = self.getTiltAngleList() - + sRate = self.inputSetOfTomograms.get().getSamplingRate() for index in range(self.getProjectionRange()): newTi = tomoObj.TiltImage() newTi.setTiltAngle(tiltAngleList[index]) - newTi.setTsId(tomoId) + newTi.setTsId(tsId) newTi.setAcquisitionOrder(index + 1) newTi.setLocation(index + 1, - os.path.join(extraPrefix, - os.path.basename(tomo.getFileName()))) - newTi.setSamplingRate(self.inputSetOfTomograms.get().getSamplingRate()) + self.getExtraOutFile(tsId, ext=MRC_EXT)) + newTi.setSamplingRate(sRate) newTs.append(newTi) ih = ImageHandler() diff --git a/imod/protocols/protocol_tomoReconstruction.py b/imod/protocols/protocol_tomoReconstruction.py index 53725667..bdf39403 100644 --- a/imod/protocols/protocol_tomoReconstruction.py +++ b/imod/protocols/protocol_tomoReconstruction.py @@ -33,7 +33,7 @@ from .. import Plugin from .protocol_base import (ProtImodBase, EXT_MRC_ODD_NAME, EXT_MRC_EVEN_NAME, - EXT_MRCS_TS_EVEN_NAME, EXT_MRCS_TS_ODD_NAME, TLT_EXT, ODD, MRCS_EXT, EVEN) + EXT_MRCS_TS_EVEN_NAME, EXT_MRCS_TS_ODD_NAME, TLT_EXT, ODD, MRCS_EXT, EVEN, MRC_EXT) class ProtImodTomoReconstruction(ProtImodBase): @@ -195,7 +195,7 @@ def convertInputStep(self, tsId, **kwargs): oddEvenFlag = self.applyToOddEven(self.inputSetOfTiltSeries.get()) super().convertInputStep(tsId, doSwap=True, oddEven=oddEvenFlag) - @ProtImodBase.tryExceptDecorator + # @ProtImodBase.tryExceptDecorator def computeReconstructionStep(self, tsId): ts = self.tsDict[tsId] paramsTilt = { @@ -263,7 +263,7 @@ def getArgs(): paramsTrimVol = { 'input': self.getTmpOutFile(tsId, ext="rec"), - 'output': self.getExtraOutFile(tsId), + 'output': self.getExtraOutFile(tsId, ext=MRC_EXT), 'options': getArgs() } @@ -284,7 +284,7 @@ def getArgs(): def createOutputStep(self, tsId): ts = self.tsDict[tsId] - tomoLocation = self.getExtraOutFile(tsId) + tomoLocation = self.getExtraOutFile(tsId, ext=MRC_EXT) if os.path.exists(tomoLocation): output = self.getOutputSetOfTomograms(self.inputSetOfTiltSeries.get()) diff --git a/imod/protocols/protocol_tsPreprocess.py b/imod/protocols/protocol_tsPreprocess.py index 3474d173..393bc674 100644 --- a/imod/protocols/protocol_tsPreprocess.py +++ b/imod/protocols/protocol_tsPreprocess.py @@ -222,9 +222,8 @@ def convertInputStep(self, tsId, **kwargs): oddEven=oddEvenFlag) @ProtImodBase.tryExceptDecorator - def generateOutputStackStep(self, tsObjId): - ts = self.inputSetOfTiltSeries.get()[tsObjId] - tsId = ts.getTsId() + def generateOutputStackStep(self, tsId): + ts = self.tsDict[tsId] firstItem = ts.getFirstItem() xfFile = None diff --git a/imod/tests/test_protocols_imod.py b/imod/tests/test_protocols_imod.py index a3140c6b..15ab217a 100644 --- a/imod/tests/test_protocols_imod.py +++ b/imod/tests/test_protocols_imod.py @@ -197,9 +197,9 @@ def _runTomoNormalization(cls, inputSetOfTomograms, binning, floatDensities, modeToOutput, scaleRangeToggle, scaleRangeMax, scaleRangeMin, meanSdToggle, - scaleMean, scaleSd, scaleMax, scaleMin) -> ProtImodTomoNormalization: + scaleMean, scaleSd, scaleMax, scaleMin) -> ProtImodTomoPreProcess: - cls.protTomoNormalization = cls.newProtocol(ProtImodTomoNormalization, + cls.protTomoNormalization = cls.newProtocol(ProtImodTomoPreProcess, inputSetOfTomograms=inputSetOfTomograms, binning=binning, floatDensities=floatDensities, @@ -449,7 +449,7 @@ def test_doseFilterOutputTS(self): tsId = ts.getFirstItem().getTsId() self.assertTrue(os.path.exists(os.path.join(self.protDoseFilter._getExtraPath(tsId), - "BB" + tsId + ".st"))) + tsId + ".st"))) def test_xRaysEraserOutputTS(self): @@ -459,7 +459,7 @@ def test_xRaysEraserOutputTS(self): tsId = ts.getFirstItem().getTsId() self.assertTrue(os.path.exists(os.path.join(self.protXRaysEraser._getExtraPath(tsId), - "BB" + tsId + ".st"))) + tsId + ".st"))) def test_excludeViewsOutputTS(self): @@ -469,7 +469,7 @@ def test_excludeViewsOutputTS(self): tsId = ts.getFirstItem().getTsId() self.assertTrue(os.path.exists(os.path.join(self.protExcludeViews._getExtraPath(tsId), - "BB" + tsId + ".st"))) + tsId + ".st"))) for index, tsOut in enumerate(ts): self.assertEqual(tsOut.getSize(), self.excludeViewsOutputSizes[tsOut.getTsId()]) @@ -482,7 +482,7 @@ def test_normalizationOutputTS(self): tsId = ts.getFirstItem().getTsId() self.assertTrue(os.path.exists(os.path.join(self.protTSNormalization._getExtraPath(tsId), - "BB" + tsId + ".st"))) + tsId + ".st"))) inSamplingRate = self.protTSNormalization.inputSetOfTiltSeries.get().getSamplingRate() outSamplingRate = ts.getSamplingRate() @@ -496,7 +496,7 @@ def test_prealignmentOutputTS(self): tsId = ts.getFirstItem().getTsId() outputLocation = os.path.join(self.protXcorr._getExtraPath(tsId), - "BB" + tsId + ".st") + tsId + ".st") self.assertTrue(os.path.exists(outputLocation)) @@ -510,7 +510,7 @@ def test_prealignmentOutputInterpolatedTS(self): tsId = ts.getFirstItem().getTsId() outputLocation = os.path.join(self.protXcorr._getExtraPath(tsId), - "BB" + tsId + ".st") + tsId + ".st") self.assertTrue(os.path.exists(outputLocation)) @@ -526,9 +526,9 @@ def test_fiducialModelstOutputFiducialModelGaps(self): tsId = output.getFirstItem().getTsId() outputLocationImod = os.path.join(self.protFiducialModels._getExtraPath(tsId), - "BB" + tsId + "_gaps.fid") + tsId + "_gaps.fid") outputLocationScipion = os.path.join(self.protFiducialModels._getExtraPath(tsId), - "BB" + tsId + "_gaps.sfid") + tsId + "_gaps.sfid") self.assertTrue(os.path.exists(outputLocationImod)) self.assertTrue(os.path.exists(outputLocationScipion)) @@ -541,7 +541,7 @@ def test_fiducialAlignmentOutputTS(self): tsId = output.getFirstItem().getTsId() outputLocation = os.path.join(self.protFiducialAlignment._getExtraPath(tsId), - "BB" + tsId + ".st") + tsId + ".st") self.assertTrue(os.path.exists(outputLocation)) @@ -554,7 +554,7 @@ def test_fiducialAlignmentOutputTS(self): tsId = output.getFirstItem().getTsId() outputLocation = os.path.join(self.protFiducialAlignment._getExtraPath(tsId), - "BB" + tsId + ".st") + tsId + ".st") self.assertTrue(os.path.exists(outputLocation)) @@ -570,7 +570,7 @@ def test_fiducialAlignmentOutputTS(self): tsId = output.getFirstItem().getTsId() outputLocation = os.path.join(self.protFiducialAlignment._getExtraPath(tsId), - "BB" + tsId + "_noGaps.sfid") + tsId + "_noGaps.sfid") self.assertTrue(os.path.exists(outputLocation)) @@ -580,7 +580,7 @@ def test_fiducialAlignmentOutputTS(self): tsId = output.getFirstItem().getTsId() outputLocation = os.path.join(self.protFiducialAlignment._getExtraPath(tsId), - "BB" + tsId + "_fid.xyz") + tsId + "_fid.xyz") self.assertTrue(os.path.exists(outputLocation)) @@ -613,7 +613,7 @@ def test_tomoReconstructionOutputTomogram(self): tomoId = self.protTomoReconstruction.inputSetOfTiltSeries.get().getFirstItem().getTsId() outputLocation = os.path.join(self.protTomoReconstruction._getExtraPath(tomoId), - "BB" + tomoId + ".mrc") + tomoId + ".mrc") self.assertTrue(os.path.exists(outputLocation)) @@ -629,10 +629,10 @@ def test_goldBeadPeaker3DOutput(self): self.assertSetSize(output, size=52, diffDelta=30) tomoId = self.protGoldBeadPicker3D.inputSetOfTomograms.get().getFirstItem().getTsId() - location = self.protGoldBeadPicker3D._getExtraPath("BB" + tomoId) + location = self.protGoldBeadPicker3D._getExtraPath(tomoId) self.assertTrue(os.path.exists(os.path.join(location, - "BB" + tomoId + ".mod"))) + tomoId + ".mod"))) def test_tomoNormalizationOutput(self): From 6b298cf7e05562a956021640950fddcfb90ae425 Mon Sep 17 00:00:00 2001 From: JorMaister Date: Thu, 11 Apr 2024 17:08:49 +0200 Subject: [PATCH 19/49] Methods formatTransformFile and genXfFile combined into one, admitting both presentAcqOrders or onlyEnabled -> backwards comp --- .../protocol_applyTransformationMatrix.py | 2 +- imod/protocols/protocol_tsPreprocess.py | 10 ++--- imod/utils.py | 41 ++++++------------- 3 files changed, 16 insertions(+), 37 deletions(-) diff --git a/imod/protocols/protocol_applyTransformationMatrix.py b/imod/protocols/protocol_applyTransformationMatrix.py index 20ac2c66..3a9472c1 100644 --- a/imod/protocols/protocol_applyTransformationMatrix.py +++ b/imod/protocols/protocol_applyTransformationMatrix.py @@ -87,7 +87,7 @@ def _initialize(self): def generateTransformFileStep(self, tsId): ts = self.tsDict[tsId] self.genTsPaths(tsId) - utils.formatTransformFile(ts, self.getExtraOutFile(tsId, ext=XF_EXT)) + utils.genXfFile(ts, self.getExtraOutFile(tsId, ext=XF_EXT)) @ProtImodBase.tryExceptDecorator def computeAlignmentStep(self, tsId): diff --git a/imod/protocols/protocol_tsPreprocess.py b/imod/protocols/protocol_tsPreprocess.py index 393bc674..ab425d13 100644 --- a/imod/protocols/protocol_tsPreprocess.py +++ b/imod/protocols/protocol_tsPreprocess.py @@ -23,18 +23,14 @@ # * e-mail address 'scipion@cnb.csic.es' # * # ***************************************************************************** - -import os from pyworkflow import BETA from pyworkflow.object import Set import pyworkflow.protocol.params as params from pwem.emlib.image import ImageHandler import tomo.objects as tomoObj - from .. import Plugin -from .protocol_base import ProtImodBase, EXT_MRCS_TS_EVEN_NAME, EXT_MRCS_TS_ODD_NAME, OUTPUT_TILTSERIES_NAME, XF_EXT, \ - ODD, EVEN, MRCS_EXT -from ..utils import formatTransformFile +from .protocol_base import ProtImodBase, OUTPUT_TILTSERIES_NAME, XF_EXT, ODD, EVEN, MRCS_EXT +from ..utils import genXfFile class ProtImodTsPreprocess(ProtImodBase): @@ -229,7 +225,7 @@ def generateOutputStackStep(self, tsId): if self.applyAlignment.get() and ts.hasAlignment(): xfFile = self.getExtraOutFile(tsId, ext=XF_EXT) - formatTransformFile(ts, xfFile) + genXfFile(ts, xfFile) binning = self.binning.get() diff --git a/imod/utils.py b/imod/utils.py index cacc3453..c0fb80ae 100644 --- a/imod/utils.py +++ b/imod/utils.py @@ -30,8 +30,7 @@ import os import csv import math -from typing import Set, Union - +from typing import Union import numpy as np import pyworkflow.object as pwobj import pyworkflow.utils as pwutils @@ -41,35 +40,15 @@ logger = logging.getLogger(__name__) -def formatTransformFile(ts, transformFilePath, onlyEnabled=False): - """ This method takes a tilt series and the output transformation file path - and creates an IMOD-based transform - file in the location indicated. """ - - tsMatrixTransformList = [] - - for ti in ts: - if onlyEnabled and not ti.isEnabled(): - continue - transform = ti.getTransform().getMatrix().flatten() - transformIMOD = ['%.7f' % transform[0], - '%.7f' % transform[1], - '%.7f' % transform[3], - '%.7f' % transform[4], - "{:>6}".format('%.3g' % transform[2]), - "{:>6}".format('%.3g' % transform[5])] - tsMatrixTransformList.append(transformIMOD) - - with open(transformFilePath, 'w') as f: - csvW = csv.writer(f, delimiter='\t') - csvW.writerows(tsMatrixTransformList) - - -def genXfFile(ts: TiltSeries, outXfName: str, presentAcqOrders: Union[set, None] = None) -> None: +def genXfFile(ts: TiltSeries, outXfName: str, + presentAcqOrders: Union[set, None] = None, + onlyEnabled: bool = False) -> None: """ This method takes a tilt series and the output transformation file path and creates an IMOD-based transform file in the location indicated. The transformation matrix of a tilt-image is only added if its acquisition order is contained in a set composed of the - acquisition orders present in both the given tilt-series and the CTFTomoSeries. + acquisition orders present in both the given tilt-series and the CTFTomoSeries. If presentAcqOrders + is not None, it is considered before the attribute onlyEnabled, as presentAcqOrders may also have been + generated considering the enabled elements of the intersection. """ def formatMatrix(tiltImage): @@ -85,7 +64,11 @@ def formatMatrix(tiltImage): if presentAcqOrders: tsMatrixList = [formatMatrix(ti) for ti in ts if ti.getAcquisitionOrder() in presentAcqOrders] else: - tsMatrixList = [formatMatrix(ti) for ti in ts] + tsMatrixList = [] + for ti in ts: + if onlyEnabled and not ti.isEnabled(): + continue + tsMatrixList.append(formatMatrix(ti)) with open(outXfName, 'w') as f: csvW = csv.writer(f, delimiter='\t') From f9ad63be0cf84f68404cdc73fe16982c26d9dafa Mon Sep 17 00:00:00 2001 From: Vilax Date: Tue, 16 Apr 2024 09:30:48 +0200 Subject: [PATCH 20/49] enhancing use and helps --- .../protocol_applyTransformationMatrix.py | 22 +- imod/protocols/protocol_ctfCorrection.py | 27 ++ .../protocol_ctfEstimation_automatic.py | 2 +- .../protocol_ctfEstimation_manual.py | 7 + imod/protocols/protocol_doseFilter.py | 20 +- imod/protocols/protocol_etomo.py | 15 +- imod/protocols/protocol_fiducialAlignment.py | 141 +++++++-- imod/protocols/protocol_fiducialModel.py | 279 ++++++++++++++---- imod/protocols/protocol_forms.py | 149 ++++++++++ imod/protocols/protocol_tomoNormalization.py | 178 ++++++----- imod/protocols/protocol_tomoProjection.py | 29 +- imod/protocols/protocol_tomoReconstruction.py | 135 ++++++--- imod/protocols/protocol_tsNormalization.py | 164 ++++++---- imod/protocols/protocol_xCorrPrealignment.py | 142 ++------- imod/protocols/protocol_xRaysEraser.py | 86 +++++- 15 files changed, 1003 insertions(+), 393 deletions(-) create mode 100644 imod/protocols/protocol_forms.py diff --git a/imod/protocols/protocol_applyTransformationMatrix.py b/imod/protocols/protocol_applyTransformationMatrix.py index bacbe4b6..ed2a1fe0 100644 --- a/imod/protocols/protocol_applyTransformationMatrix.py +++ b/imod/protocols/protocol_applyTransformationMatrix.py @@ -40,8 +40,15 @@ class ProtImodApplyTransformationMatrix(ProtImodBase): """ Compute the interpolated tilt-series from its transform matrix. + The protocol makes use of the IMod command newstack More info: https://bio3d.colorado.edu/imod/doc/man/newstack.html + + Generally, the tilt series has an associated transformation matrix + which contains the alignment information. The transformation matrix + is usually associated but not applied to avoid to accumulate interpolation + errors during the image processing. This protocol allows to apply + the transformation matrix to the tilt series """ _label = 'Apply transformation' @@ -54,15 +61,18 @@ def _defineParams(self, form): params.PointerParam, pointerClass='SetOfTiltSeries', important=True, - label='Input set of tilt-series') + label='Input set of tilt-series to applied the transformation matrix') form.addParam('binning', params.IntParam, default=1, - label='Binning', - help='Binning to be applied to the interpolated ' - 'tilt-series in IMOD convention. Images will be ' - 'binned by the given factor. Must be an integer ' - 'bigger than 1') + label='Binning for the interpolated', + help='Binning to be applied to the interpolated tilt-series in IMOD ' + 'convention. \n' + 'Binning is an scaling factor given by an integer greater than 1. ' + 'IMOD uses ordinary binning to reduce images in size by the given factor. ' + 'The value of a binned pixel is the average of pixel values in each block ' + 'of pixels being binned. Binning is applied before all other image ' + 'transformations.') form.addParam('processOddEven', params.BooleanParam, diff --git a/imod/protocols/protocol_ctfCorrection.py b/imod/protocols/protocol_ctfCorrection.py index 2d730c1e..e9f2bc4c 100644 --- a/imod/protocols/protocol_ctfCorrection.py +++ b/imod/protocols/protocol_ctfCorrection.py @@ -42,6 +42,33 @@ class ProtImodCtfCorrection(ProtImodBase): CTF correction of a set of input tilt-series using the IMOD procedure. More info: https://bio3d.colorado.edu/imod/doc/man/ctfphaseflip.html + + This program will correct the CTF of an input tilt series by phase + flipping, with an option to attenuate frequencies near the zeros of the + CTF. + + Ctfphaseflip corrects each view strip by strip. A strip is defined as + an image region whose defocus difference is less than a user specified + value, the defocus tolerance. Normally, the strips are vertically ori- + ented and defocus is assumed to be the same along a vertical line. + Thus, the tilt series must be aligned so that the tilt axis is vertical + before applying this correction. The original thinking was that an + image region with defocus difference less than the tolerance could be + considered to have constant defocus and could be corrected as one + strip. However, the validity of the correction at the center of the + strip probably does not depend on whether it contains material beyond + this focus range, since only vertical lines near or at the center are + used in the corrected image. The program may limit the width further + to reduce computation time, or expand it to retain enough resolution + between successive zeros in the X direction of frequency space. + + Through most of the image, each strip is corrected based on the defocus + at the center of the strip. However, the strips at the left and right + edges of the image may be corrected repeatedly, at different defocus + values, in order to extend the correction close enough to the edges of + the image. + + """ _label = 'CTF correction' diff --git a/imod/protocols/protocol_ctfEstimation_automatic.py b/imod/protocols/protocol_ctfEstimation_automatic.py index 6b57507f..f61513e9 100644 --- a/imod/protocols/protocol_ctfEstimation_automatic.py +++ b/imod/protocols/protocol_ctfEstimation_automatic.py @@ -55,7 +55,7 @@ def _defineParams(self, form): form.addParam('inputSet', params.PointerParam, pointerClass='SetOfTiltSeries, SetOfCTFTomoSeries', - label='Input set of tilt-series', + label='Tilt-series', help='This should be a *raw stack*, not an aligned stack, ' 'because the interpolation used to make ' 'an aligned stack attenuates high frequencies and ' diff --git a/imod/protocols/protocol_ctfEstimation_manual.py b/imod/protocols/protocol_ctfEstimation_manual.py index 10b9b74d..95fb7b07 100644 --- a/imod/protocols/protocol_ctfEstimation_manual.py +++ b/imod/protocols/protocol_ctfEstimation_manual.py @@ -40,6 +40,13 @@ class ProtImodManualCtfEstimation(ProtImodAutomaticCtfEstimation): More info: https://bio3d.colorado.edu/imod/doc/man/ctfplotter.html + This GUI program will plot the logarithm of a rotationally averaged + power spectrum of an input tilt series after subtracting the noise + floor. The method is based on periodogram averaging; namely, averaging + of spectra from small, overlapping areas of the images, referred to as + tiles. The user can interactively choose which projection views are + included in the averaging. It is also possible to run the program non- + interactively for automatic fitting. """ _label = 'CTF estimation (manual)' diff --git a/imod/protocols/protocol_doseFilter.py b/imod/protocols/protocol_doseFilter.py index fa2d1318..b5f240d3 100644 --- a/imod/protocols/protocol_doseFilter.py +++ b/imod/protocols/protocol_doseFilter.py @@ -45,6 +45,16 @@ class ProtImodDoseFilter(ProtImodBase): Tilt-series dose filtering based on the IMOD procedure. More info: https://bio3d.colorado.edu/imod/doc/man/mtffilter.html + + A specialized filter can be applied to perform dose weight-filtering of + cryoEM images, particularly ones from tilt series. The filter is as + described in Grant and Grigorieff, 2015 (DOI: 10.7554/eLife.06980) and + the implementation follows that in their "unblur" program. At any fre- + quency, the filter follows an exponential decay with dose, where the + exponential is of the dose divided by 2 times a "critical dose" for + that frequency. This critical dose was empirically found to be approx- + imated by a * k^b + c, where k is frequency; the values of a, b, c in + that paper are used by default. """ _label = 'Dose filter' @@ -58,13 +68,15 @@ def _defineParams(self, form): params.PointerParam, pointerClass='SetOfTiltSeries', important=True, - label='Input set of tilt-series') + label='Tilt Series', + help='This input tilt-series will be low pass filtered according' + 'to their cumulated dose') form.addParam('initialDose', params.FloatParam, default=0.0, expertLevel=params.LEVEL_ADVANCED, - label='Initial dose (e/sq. Å)', + label='Initial dose (e/Å^2)', help='Dose applied before any of the images in the ' 'input file were taken; this value will be ' 'added to all the dose values.') @@ -81,12 +93,12 @@ def _defineParams(self, form): 'during import of the tilt-series\n' '- Fixed dose: manually input fixed dose ' 'for each image of the input file, ' - 'in electrons/square Ångstrom.') + 'in electrons/Å^2.') form.addParam('fixedImageDose', params.FloatParam, default=FIXED_DOSE, - label='Fixed dose (e/sq Å)', + label='Fixed dose (e/Å^2)', condition='inputDoseType == %i' % FIXED_DOSE, help='Fixed dose for each image of the input file, ' 'in electrons/square Ångstrom.') diff --git a/imod/protocols/protocol_etomo.py b/imod/protocols/protocol_etomo.py index 0a05ef2c..bbc434d1 100644 --- a/imod/protocols/protocol_etomo.py +++ b/imod/protocols/protocol_etomo.py @@ -46,6 +46,17 @@ class ProtImodEtomo(ProtImodBase): More info: https://bio3d.colorado.edu/imod/doc/etomoTutorial.html + + Etomo is software tool for assisting users in the tomographic reconstruction + process of both single and dual axis tilt series. Throughout this procedure, + eTomo executes numerous program commands and frequently initiates 3dmod + and Midas to enable users to make precise adjustments. Some of the main features + are:\n + - Xray eraser\n + - dose filtering\n + - Tilt series alignment\n + - Gold beads detection and eraser\n + - Tomogram reconstruction\n """ _label = 'Etomo interactive' @@ -73,7 +84,9 @@ def _defineParams(self, form): params.FloatParam, default=10, label='Fiducial markers diameter (nm)', - help='Diameter of gold beads in nanometers.') + help='Diameter of gold beads in nanometers. Note that fiducials are' + 'small gold beads that are used as marker elements in the images.' + 'They can be used as reference points to align the tilt series') form.addParam('applyAlignment', params.BooleanParam, diff --git a/imod/protocols/protocol_fiducialAlignment.py b/imod/protocols/protocol_fiducialAlignment.py index f261fbc0..758bc5f3 100644 --- a/imod/protocols/protocol_fiducialAlignment.py +++ b/imod/protocols/protocol_fiducialAlignment.py @@ -45,10 +45,70 @@ class ProtImodFiducialAlignment(ProtImodBase): on the IMOD procedure. More info: https://bio3d.colorado.edu/imod/doc/man/tiltalign.html - https://bio3d.colorado.edu/imod/doc/man/model2point.html - https://bio3d.colorado.edu/imod/doc/man/imodtrans.html - https://bio3d.colorado.edu/imod/doc/man/newstack.html - https://bio3d.colorado.edu/imod/doc/man/ccderaser.html + + This program will solve for the displacements, rotations, tilts, and + magnification differences relating a set of tilted views of an object. + It uses a set of fiducial points that have been identified in a series + of views. These input data are read from a model in which each fiducial + point is a separate contour. + + This program has several notable features: + + 1) Any given fiducial point need not be present in every view. Thus, + one can track each fiducial point only through the set of views in + which it can be reliably identified, and one can even skip views in the + middle of that set. + + 2) The program can solve for distortion (stretching) in the plane of + the section. + + 3) It is possible to constrain several views to have the same unknown + value of rotation, tilt angle, magnification, compression, or distor- + tion. This can reduce the number of unknowns and can give more accu- + rate overall solutions. + + 4) If the fiducial points are supposed to lie in one or two planes, + then after the minimization procedure is complete, the program can ana- + lyze the solved point positions and determine the slope of this plane. + It uses this slope to estimate how to adjust tilt angles so as to make + the planes be horizontal in a reconstruction. + + 5) The program can use a robust fitting method to give different + weights to different modeled points based on their individual fitting + errors. Points with the most extreme errors are eliminated from the + fit, and ones with high but less extreme errors are down-weighted. + This fitting provides a substitute for fixing many modeled point posi- + tions based on their errors. + + _On the alignment model_:\n + The program implements the following model for the imaging of the spec- + imen in each individual view: + 1) The specimen itself changes by + a) an isotropic size change (magnification variable); + b) additional thinning in the Z dimension (compression variable); and + c) linear stretch along one axis in the specimen plane, implemented by + variables representing stretch along the X axis and skew between + the X and Y axes; + 2) The specimen is tilted slightly around the X axis (X tilt variable) + 3) The specimen is tilted around the X axis by the negative of the beam + tilt, if any (one variable for all views) + 4) The specimen is tilted around the Y axis (tilt variable) + 5) The specimen is tilted back around the X axis by the beam tilt, if any + 6) The projected image rotates in the plane of the camera (rotation + variable) + 7) The projected image may stretch along an axis midway between the + original X and Y axes (one variable for all views) + 8) The image shifts on the camera + + The complete model is summarized in: + Mastronarde, D. N. 2008. Correction for non-perpendicularity of beam + and tilt axis in tomographic reconstructions with the IMOD package. J. + Microsc. 230: 212-217. + The version of the model prior to the addition of beam tilt is described + in more detail in: + Mastronarde, D. N. 2007. Fiducial marker and hybrid alignment methods + for single- and double-axis tomography. In: Electron Tomography, Ed. + J. Frank, 2nd edition, pp 163-185. Springer, New York. """ _label = 'Fiducial alignment' @@ -63,7 +123,7 @@ def _defineParams(self, form): params.PointerParam, pointerClass='SetOfLandmarkModels', important=True, - label='Input set of fiducial models.') + label='Fiducial model') # TODO: Allow for a different set of tilt-series input source than the one from the landmark model. This is not # TODO: possible due to a change of data type when applying the transformation with scipion applyTransform @@ -91,17 +151,14 @@ def _defineParams(self, form): params.EnumParam, choices=['Yes', 'No'], default=1, - label='Find beads on two surfaces?', + label='Assume beads on two surfaces?', display=params.EnumParam.DISPLAY_HLIST, - help="Track fiducials differentiating in which side of " - "the sample are located.\nIMPORTANT: It is highly " - "recommended to match the option selected in the " - "generation of the fiducial models. In case they " - "do not match, it is not intended to fail but could " - "be missing the whole potential of the algorithm. " - "In case the algorithm used fot he calculation" - "of the fiducial models does not consider this " - "option it is algo recomended to set this " + help="Track fiducials differentiating in which side of the sample are located.\n" + "IMPORTANT: It is highly recommended to match the option selected in the " + "generation of the fiducial models. In case they do not match, it is not " + "intended to fail but could be missing the whole potential of the algorithm. " + "In case the algorithm used for the calculation of the fiducial models does " + "not consider this option it is algo recommended to set this " "option to 'No'.") form.addParam('computeAlignment', @@ -111,8 +168,12 @@ def _defineParams(self, form): label='Generate interpolated tilt-series?', important=True, display=params.EnumParam.DISPLAY_HLIST, - help='Generate and save the interpolated tilt-series ' - 'applying the obtained transformation matrices.') + help='Generate and save the interpolated tilt-series applying the obtained transformation ' + 'matrices.\n' + 'By default, the output of this protocol will be a tilseries that will have associated' + 'the alignment information as a transformation matrix. When this option is set as Yes, ' + 'then a second output, called interpolated tilt series, is generated. The interpolated tilt ' + 'series should be used for visualization purpose but not for image processing') groupInterpolation = form.addGroup('Interpolated tilt-series', condition='computeAlignment==0') @@ -120,12 +181,14 @@ def _defineParams(self, form): groupInterpolation.addParam('binning', params.IntParam, default=1, - label='Binning', - help='Binning to be applied to the ' - 'interpolated tilt-series in IMOD ' - 'convention. Images will be binned ' - 'by the given factor. Must be an ' - 'integer bigger than 1') + label='Binning for the interpolated', + help='Binning to be applied to the interpolated tilt-series in IMOD ' + 'convention. \n' + 'Binning is an scaling factor given by an integer greater than 1. ' + 'IMOD uses ordinary binning to reduce images in size by the given factor. ' + 'The value of a binned pixel is the average of pixel values in each block ' + 'of pixels being binned. Binning is applied before all other image ' + 'transformations.') form.addSection('Global variables') @@ -136,14 +199,19 @@ def _defineParams(self, form): default=3, label='Rotation solution type', display=params.EnumParam.DISPLAY_HLIST, - help='Type of rotation solution.') + help='Type of rotation solution: See rotOption in tiltalign IMOD command \n' + '* No rotation: The in-plane rotation will not be estimated\n' + '* One rotation: To solve for a single rotation variable \n' + '* Group rotations: Group views to solve for fewer rotations variables. Automapping of ' + 'rotation variables linearly changing values\n' + '* Solve for all rotations: for each view having an independent rotation\n') form.addParam('groupRotationSize', params.IntParam, default=5, condition='rotationSolutionType==2', label='Group size', - help='Size of the rotation group') + help='Default group size when automapping rotation variables') form.addParam('magnificationSolutionType', params.EnumParam, @@ -153,14 +221,18 @@ def _defineParams(self, form): default=1, label='Magnification solution type', display=params.EnumParam.DISPLAY_HLIST, - help='Type of magnification solution.') + help='Type of magnification solution: See MagOption in tiltaling IMOD command\n' + '* Fixed magnification: Do not solve magnification. This fixes all magnifications at 1.0.\n' + '* Group magnifications: Group views to solve for fewer magnifications variables. ' + 'Automapping of variables (linearly changing values) \n' + '* Solve for all magnifications: to vary all magnifications of each view independently\n') form.addParam('groupMagnificationSize', params.IntParam, default=4, condition='magnificationSolutionType==1', label='Group size', - help='Size of the magnification group') + help='Group size when automapping magnification variables') form.addParam('tiltAngleSolutionType', params.EnumParam, @@ -169,14 +241,18 @@ def _defineParams(self, form): default=1, label='Tilt angle solution type', display=params.EnumParam.DISPLAY_HLIST, - help='Type of tilt angle solution.') + help='Type of tilt angle solution: See TiltOption in tiltalign IMOD command\n' + ' * Fixed tilt angles: To fix all tilt angles at their initial (input) values \n' + ' * Group tilt angles: To automap groups of tilt angles (linearly changing values) \n' + ' * Solve for all except minimum tilt:to solve for all tilt angles except for the view ' + 'at minimum tilt \n') form.addParam('groupTiltAngleSize', params.IntParam, default=5, condition='tiltAngleSolutionType==1', label='Group size', - help='Size of the tilt angle group') + help='Average default group size when automapping tilt variables') form.addParam('distortionSolutionType', params.EnumParam, @@ -184,7 +260,12 @@ def _defineParams(self, form): default=0, label='Distortion solution type', display=params.EnumParam.DISPLAY_HLIST, - help='Type of distortion solution.') + help='Type of skew solution:' + '* 0 to fix all skew angles at 0.0 \n' + '* 1 to vary all skew angles independently\n ' + '* 2 to specify a mapping of skew variables, or\n ' + '* 3 or 4 for automapping of variables (3 for linearly changing values or 4 for values all ' + 'the same within a group)..') form.addParam('xStretchGroupSize', params.IntParam, diff --git a/imod/protocols/protocol_fiducialModel.py b/imod/protocols/protocol_fiducialModel.py index 714f6ed4..03eda9de 100644 --- a/imod/protocols/protocol_fiducialModel.py +++ b/imod/protocols/protocol_fiducialModel.py @@ -34,12 +34,14 @@ from .. import Plugin, utils from .protocol_base import ProtImodBase +from .protocol_forms import CommonIMODforms -class ProtImodFiducialModel(ProtImodBase): +class ProtImodFiducialModel(ProtImodBase, CommonIMODforms): """ Construction of a fiducial model and alignment of tilt-series based on the IMOD procedure. + More info: https://bio3d.colorado.edu/imod/doc/man/autofidseed.html https://bio3d.colorado.edu/imod/doc/man/beadtrack.html @@ -49,65 +51,123 @@ class ProtImodFiducialModel(ProtImodBase): _label = 'Generate fiducial model' _devStatus = BETA + FIDUCIAL_MODEL = 0 + PATCH_TRACKING = 1 + # -------------------------- DEFINE param functions ----------------------- def _defineParams(self, form): form.addSection('Input') + form.addParam('typeOfModel', + params.EnumParam, + choices=["Make seed and Track", "Patch Tracking"], + default=0, + important=True, + label='Model generation') + form.addParam('inputSetOfTiltSeries', params.PointerParam, pointerClass='SetOfTiltSeries', important=True, - label='Input set of tilt-series') - - form.addParam('fiducialDiameter', - params.FloatParam, - label='Fiducial diameter (nm)', - default='10', - important=True, - help="Fiducials diameter to be tracked for alignment.") - - form.addParam('twoSurfaces', - params.EnumParam, - choices=['Yes', 'No'], - default=1, - label='Find beads on two surfaces?', - display=params.EnumParam.DISPLAY_HLIST, - help="Track fiducials differentiating in which side " - "of the sample are located.") - - form.addParam('numberFiducial', - params.IntParam, - label='Number of fiducials', - default='25', - expertLevel=params.LEVEL_ADVANCED, - help="Number of fiducials to be tracked for alignment.") - - form.addParam('doTrackWithModel', params.BooleanParam, - default=True, - label="Track with fiducial model as seed", - help="Turn the tracked model into new seed and " - "repeat tracking.") - - form.addParam('shiftsNearZeroFraction', - params.FloatParam, - label='Shifts near zero fraction', - default='0.2', - expertLevel=params.LEVEL_ADVANCED, - help="Fraction of the tracking box size above which to " - "supply shifts near zero tilt to Beadtrack. The " - "dominant net shifts in the bead positions between " - "views are found as described above, and if one of " - "the shifts is larger than this fraction of the " - "-BoxSizeXandY entry to Beadtrack, then the shifts " - "are provided when running Beadtrack on the initial " - "seed models. Also, a command file will be written " - "with modified parameters, named as the root name " - "of the input command file followed by '_adjusted' " - "and its extension. Enter 0 or a large value to " - "disable this analysis.") + label='Tilt Series') + + self._patchTrackingForm(form, 'typeOfModel==%d' % self.PATCH_TRACKING) + self._fiducialSeedForm(form, 'typeOfModel==%d' % self.FIDUCIAL_MODEL) + + def _patchTrackingForm(self, form, condition, levelType=params.LEVEL_NORMAL): + patchtrack = form.addGroup('Patch Tracking', expertLevel=levelType, condition=condition) + + patchtrack.addParam('sizeOfPatches', params.NumericListParam, + label='Size of the patches (X,Y)', + allowsNull=True, + expertLevel=levelType, + help="Size of the patches to track by correlation. In imod documentation " + "(tiltxcorr: SizeOfPatchesXandY)") + + patchtrack.addParam('patchLayout', + params.EnumParam, + choices=['Fractional overlap of patches', + 'Number of patches'], + default=0, + label='Patch layout', + display=params.EnumParam.DISPLAY_HLIST, + help='To be added') + + patchtrack.addParam('overlapPatches', + params.NumericListParam, + default='0.33 0.33', + condition='patchLayout==0', + label='Fractional overlap of the patches (X,Y)', + help="Fractional overlap in X and Y to track by correlation. In imod documentation" + "(tiltxcorr: NumberOfPatchesXandY)") + + patchtrack.addParam('numberOfPatches', + params.NumericListParam, + condition='patchLayout==1', + label='Number of patches (X,Y)', + help="Number of patches in X and Y of the patches. In imod documentation" + "(tiltxcorr: OverlapOfPatchesXandY)") + + patchtrack.addParam('iterationsSubpixel', + params.IntParam, + label='Iterations to increase subpixel accuracy', + help="Number of iteration of each correlation to reduce interpolation of the peak position" + "In imod documentation: (tiltxcorr: IterateCorrelations)") + + self.trimimgForm(patchtrack, pxTrimCondition='True', correlationCondition='True',levelType=params.LEVEL_ADVANCED) + self.filteringParametersForm(form, condition=condition, levelType=params.LEVEL_ADVANCED) + + def _fiducialSeedForm(self, form, condition, levelType=params.LEVEL_NORMAL): + seedModel = form.addGroup('"Make seed and Track', expertLevel=levelType, condition=condition) + seedModel.addParam('fiducialDiameter', + params.FloatParam, + label='Fiducial diameter (nm)', + default='10', + important=True, + help="Fiducials diameter to be tracked for alignment.") + + seedModel.addParam('twoSurfaces', + params.EnumParam, + choices=['Yes', 'No'], + default=1, + label='Find beads on two surfaces?', + display=params.EnumParam.DISPLAY_HLIST, + help="Track fiducials differentiating in which side " + "of the sample are located.") + + seedModel.addParam('numberFiducial', + params.IntParam, + label='Number of fiducials', + default='25', + expertLevel=params.LEVEL_ADVANCED, + help="Number of fiducials to be tracked for alignment.") + + seedModel.addParam('doTrackWithModel', params.BooleanParam, + default=True, + label="Track with fiducial model as seed", + help="Turn the tracked model into new seed and " + "repeat tracking.") + + seedModel.addParam('shiftsNearZeroFraction', + params.FloatParam, + label='Shifts near zero fraction', + default='0.2', + expertLevel=params.LEVEL_ADVANCED, + help="Fraction of the tracking box size above which to " + "supply shifts near zero tilt to Beadtrack. The " + "dominant net shifts in the bead positions between " + "views are found as described above, and if one of " + "the shifts is larger than this fraction of the " + "-BoxSizeXandY entry to Beadtrack, then the shifts " + "are provided when running Beadtrack on the initial " + "seed models. Also, a command file will be written " + "with modified parameters, named as the root name " + "of the input command file followed by '_adjusted' " + "and its extension. Enter 0 or a large value to " + "disable this analysis.") groupGlobalVariables = form.addGroup('Filter variables', - expertLevel=params.LEVEL_ADVANCED) + expertLevel=params.LEVEL_ADVANCED, condition=condition) groupGlobalVariables.addParam('refineSobelFilter', params.EnumParam, @@ -141,9 +201,13 @@ def _insertAllSteps(self): for ts in self.inputSetOfTiltSeries.get(): tsObjId = ts.getObjId() self._insertFunctionStep(self.convertInputStep, tsObjId) - self._insertFunctionStep(self.generateTrackComStep, tsObjId) - self._insertFunctionStep(self.generateFiducialSeedStep, tsObjId) - self._insertFunctionStep(self.generateFiducialModelStep, tsObjId) + if self.typeOfModel == self.FIDUCIAL_MODEL: + self._insertFunctionStep(self.generateTrackComStep, tsObjId) + self._insertFunctionStep(self.generateFiducialSeedStep, tsObjId) + self._insertFunctionStep(self.generateFiducialModelStep, tsObjId) + else: + self._insertFunctionStep(self.xcorrStep, tsObjId) + self._insertFunctionStep(self.chopcontsStep, tsObjId) self._insertFunctionStep(self.translateFiducialPointModelStep, tsObjId) self._insertFunctionStep(self.computeOutputModelsStep, tsObjId) self._insertFunctionStep(self.createOutputFailedSet, tsObjId) @@ -421,6 +485,115 @@ def computeOutputModelsStep(self, tsObjId): output.update(landmarkModelGaps) output.write() + def xcorrStep(self, tsObjId): + ''' + Imod uses the next command line for the xcorr alignment + $tiltxcorr -StandardInput + InputFile cryo_preali.mrc + OutputFile cryo_pt.fid + RotationAngle -12.6 + TiltFile cryo.rawtlt + FilterRadius2 0.125 + FilterSigma1 0.03 + FilterSigma2 0.03 + BordersInXandY 102,102 + IterateCorrelations 1 + SizeOfPatchesXandY 680,680 + OverlapOfPatchesXandY 0.33,0.33 + PrealignmentTransformFile cryo.prexg + ImagesAreBinned 1 + ''' + ts = self._getTiltSeries(tsObjId) + tsId = ts.getTsId() + extraPrefix = self._getExtraPath(tsId) + tmpPrefix = self._getTmpPath(tsId) + + firstItem = ts.getFirstItem() + angleFilePath = os.path.join(tmpPrefix, firstItem.parseFileName(extension=".tlt")) + + xfFile = os.path.join(tmpPrefix, firstItem.parseFileName(extension=".xf")) + ts.writeXfFile(xfFile) + + borders = self.pxTrim.getListFromValues() + sizePatches = self.sizeOfPatches.getListFromValues() + + + BordersInXandY = '%d,%d' % (borders[0], borders[1]) + SizeOfPatchesXandY = '%d,%d' % (sizePatches[0], sizePatches[1]) + + paramsTiltXCorr = { + 'inputFile': os.path.join(tmpPrefix, firstItem.parseFileName()), + 'outputFile': os.path.join(extraPrefix, firstItem.parseFileName(suffix="_pt", extension=".fid")), + 'RotationAngle': ts.getAcquisition().getTiltAxisAngle(), + 'TiltFile': angleFilePath, + 'FilterRadius2': self.filterRadius2.get(), + 'FilterSigma1': self.filterSigma1.get(), + 'FilterSigma2': self.filterSigma2.get(), + 'BordersInXandY': BordersInXandY, + 'IterateCorrelations': self.iterationsSubpixel.get(), + 'SizeOfPatchesXandY': SizeOfPatchesXandY, + 'PrealignmentTransformFile': xfFile, + + 'ImagesAreBinned': 1, + } + argsTiltXCorr = " " \ + "-InputFile %(inputFile)s " \ + "-OutputFile %(outputFile)s " \ + "-RotationAngle %(RotationAngle)s " \ + "-TiltFile %(TiltFile)s " \ + "-FilterRadius2 %(FilterRadius2)s " \ + "-FilterSigma1 %(FilterSigma1)s " \ + "-FilterSigma2 %(FilterSigma2)s " \ + "-BordersInXandY %(BordersInXandY)s " \ + "-IterateCorrelations %(IterateCorrelations)s " \ + "-SizeOfPatchesXandY %(SizeOfPatchesXandY)s " \ + "-PrealignmentTransformFile %(PrealignmentTransformFile)s " \ + "-ImagesAreBinned %(ImagesAreBinned)s " + + if self.patchLayout.get() == 0: + patchesXY = self.overlapPatches.getListFromValues(caster=float) + OverlapOfPatchesXandY = '%f,%f' % (patchesXY[0], patchesXY[1]) + argsTiltXCorr += ' -OverlapOfPatchesXandY %s ' % OverlapOfPatchesXandY + else: + numberPatchesXY = self.numberOfPatches.getListFromValues() + argsTiltXCorr += ' -NumberOfPatchesXandY %d,%d ' % (numberPatchesXY[0], numberPatchesXY[1]) + + Plugin.runImod(self, 'tiltxcorr', argsTiltXCorr % paramsTiltXCorr) + + def chopcontsStep(self, tsObjId): + ''' + $imodchopconts -StandardInput + InputModel cryo_pt.fid + OutputModel cryo.fid + MinimumOverlap 4 + AssignSurfaces 1 + ''' + + ts = self._getTiltSeries(tsObjId) + tsId = ts.getTsId() + extraPrefix = self._getExtraPath(tsId) + + firstItem = ts.getFirstItem() + MinimumOverlap = 4 + AssignSurfaces = 1 + LengthOfPieces = -1 + + paramschopconts = { + 'inputFile': os.path.join(extraPrefix, firstItem.parseFileName(suffix="_pt", extension=".fid")), + 'outputFile': os.path.join(extraPrefix, firstItem.parseFileName(suffix="_gaps", extension=".fid")), + 'MinimumOverlap': MinimumOverlap, + 'AssignSurfaces': AssignSurfaces, + 'LengthOfPieces': LengthOfPieces + } + argschopconts = " " \ + "-InputModel %(inputFile)s " \ + "-OutputModel %(outputFile)s " \ + "-MinimumOverlap %(MinimumOverlap)s " \ + "-AssignSurfaces %(AssignSurfaces)s " \ + "-LengthOfPieces %(LengthOfPieces)s " + + Plugin.runImod(self, 'imodchopconts', argschopconts % paramschopconts) + def createOutputStep(self): if self.FiducialModelGaps: self.FiducialModelGaps.setStreamState(Set.STREAM_CLOSED) diff --git a/imod/protocols/protocol_forms.py b/imod/protocols/protocol_forms.py new file mode 100644 index 00000000..5960923b --- /dev/null +++ b/imod/protocols/protocol_forms.py @@ -0,0 +1,149 @@ +# ***************************************************************************** +# * +# * Authors: Federico P. de Isidro Gomez (fp.deisidro@cnb.csic.es) [1] +# * +# * [1] Centro Nacional de Biotecnologia, CSIC, Spain +# * +# * This program is free software; you can redistribute it and/or modify +# * it under the terms of the GNU General Public License as published by +# * the Free Software Foundation; either version 3 of the License, or +# * (at your option) any later version. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, write to the Free Software +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +# * 02111-1307 USA +# * +# * All comments concerning this program package may be sent to the +# * e-mail address 'scipion@cnb.csic.es' +# * +# ***************************************************************************** + +import pyworkflow.protocol.params as params + + +class CommonIMODforms: + def trimimgForm(self, form, pxTrimCondition='False', correlationCondition='True', levelType=params.LEVEL_ADVANCED): + ''' + Generally, this form will be integrated in a groupForm, the group form argument is form. A set of flags + control what elements are shown + ''' + form.addParam('pxTrim', + params.NumericListParam, + condition=pxTrimCondition, + label='Pixels to trim (x y without coma separator)', + default="0 0", + help='Pixels to trim off each side in X and Y.\n' + 'Some trimming should be used for patch tracking', + expertLevel=levelType) + + xtrimming = form.addLine('Pixels to do correlation along X-axis', + expertLevel=levelType, + condition=correlationCondition, + help="Starting and ending X coordinates of a region to correlate, " + "based on the position of the region at zero tilt.") + + xtrimming.addParam('xmin', + params.IntParam, + label='X axis min (left)', + allowsNull=True, + expertLevel=levelType) + + xtrimming.addParam('xmax', + params.IntParam, + label='X axis max (right)', + allowsNull=True, + expertLevel=levelType) + + ytrimming = form.addLine('Pixels to do correlation along Y-axis', + expertLevel=levelType, + condition=correlationCondition, + help="Starting and ending Y coordinates of a region to correlate, " + "based on the position of the region at zero tilt.") + + ytrimming.addParam('ymin', + params.IntParam, + label='Y axis min (top)', + allowsNull=True, + expertLevel=levelType) + + ytrimming.addParam('ymax', + params.IntParam, + label='Y axis max (botton)', + allowsNull=True, + expertLevel=levelType) + + + def filteringParametersForm(self, form, condition, levelType=params.LEVEL_NORMAL): + filtering = form.addGroup('Filtering parameters', + condition=condition, + expertLevel=levelType) + + line1 = filtering.addLine('High pass filter', + expertLevel=levelType, + help="Some high pass filtering, using a small value of Sigma1 such " + "as 0.03, may be needed to keep the program from being misled by very " + "large scale features in the images. If the images are noisy, some low " + "pass filtering with Sigma2 and Radius2 is appropriate (e.g. 0.05 for " + " Sigma2, 0.25 for Radius2). If the images are binned, these values " + "specify frequencies in the binned image, so a higher cutoff (less filtering) " + "might be appropriate.\n\n" + "" + "*FilterRadius1*: Low spatial frequencies in the cross-correlation " + "will be attenuated by a Gaussian curve that is 1 " + "at this cutoff radius and falls off below this " + "radius with a standard deviation specified by " + "FilterSigma2. Spatial frequency units range from " + "0 to 0.5.\n" + "*Filter sigma 1*: Sigma value to filter low frequencies in the " + "correlations with a curve that is an inverted " + "Gaussian. This filter is 0 at 0 frequency and " + "decays up to 1 with the given sigma value. " + "However, if a negative value of radius1 is entered, " + "this filter will be zero from 0 to " + "|radius1| then decay up to 1.") + + line1.addParam('filterRadius1', + params.FloatParam, + label='Filter radius 1', + default='0.0', + expertLevel=levelType) + + line1.addParam('filterSigma1', + params.FloatParam, + label='Filter sigma 1', + default='0.03', + expertLevel=levelType) + + line2 = filtering.addLine('Low pass filter', + expertLevel=levelType, + help="If the images are noisy, some low " + "pass filtering with Sigma2 and Radius2 is appropriate (e.g. 0.05 for " + " Sigma2, 0.25 for Radius2). If the images are binned, these values " + "specify frequencies in the binned image, so a higher cutoff (less filtering) " + "might be appropriate.\n\n" + "*Filter radius 2*: High spatial frequencies in the cross-correlation " + "will be attenuated by a Gaussian curve that is 1 " + "at this cutoff radius and falls off above this " + "radius with a standard deviation specified by " + "FilterSigma2.\n" + "*Filter sigma 2*: Sigma value for the Gaussian rolloff below and " + "above the cutoff frequencies specified by " + "FilterRadius1 and FilterRadius2") + + line2.addParam('filterRadius2', + params.FloatParam, + label='Filter radius 2', + default='0.25', + expertLevel=levelType) + + line2.addParam('filterSigma2', + params.FloatParam, + label='Filter sigma 2', + default='0.05', + expertLevel=levelType) diff --git a/imod/protocols/protocol_tomoNormalization.py b/imod/protocols/protocol_tomoNormalization.py index f1b0985f..ee7423b3 100644 --- a/imod/protocols/protocol_tomoNormalization.py +++ b/imod/protocols/protocol_tomoNormalization.py @@ -39,9 +39,27 @@ class ProtImodTomoNormalization(ProtImodBase): """ Normalize input tomogram and change its storing formatting. + More info: https://bio3D.colorado.edu/imod/doc/newstack.html - https://bio3D.colorado.edu/imod/doc/binvol.html + https://bio3d.colorado.edu/imod/doc/man/binvol.html + + IMOD tilt series preprocess makes use of the Newstack and + binvol commands. In particular, three functionalities are possible:\n + + _1 Binning_: Binvol will bin down a volume in all three dimensions, + with the binning done isotropically. Binning means summing (actually + averaging) all of the values in a block of voxels (e.g., 2x2x2 + or 1x1x3) in the input volume to create one voxel in the output volume. + The output file will have appropriately larger pixel spacings + in its header.\n + _2 Normalization_: This protocol allows to scale the gray values + of the tomograms, also called normalization, to a common range or + mean of density. The most used normalization consists in zero + mean and standard deviation one.\n + + _3 storage format_: IMOD is able to modify the number of bit of + the stored data in order to reduce the disc occupancy. """ _label = 'Tomo preprocess' @@ -55,50 +73,117 @@ def _defineParams(self, form): params.PointerParam, pointerClass='SetOfTomograms', important=True, - label='Input set of tomograms') + label='Input set of tomograms', + help='Introduce the set tomograms to be normalized') form.addParam('binning', params.IntParam, default=1, label='Binning', important=True, - help='Binning to be applied to the normalized tomograms ' - 'in IMOD convention. Volumes will be binned by the ' - 'given factor. Must be an integer bigger than 1') + help='Binning is an scaling factor for the output tomograms. ' + 'Must be an integer greater than 1. IMOD uses ordinary' + 'binning to reduce tomograms in size by the given factor. ' + 'The value of a binned pixel is the average of pixel ' + 'values in each block of pixels being binned. Binning ' + 'is applied before all') form.addParam('floatDensities', params.EnumParam, - choices=['default', '1', '2', '3', '4'], + choices=['No adjust', + 'range between min and max', + 'scaled to common mean and standard deviation', + 'shifted to a common mean without scaling', + 'shifted to mean and rescaled to a min and max'], default=0, label='Adjust densities mode', - display=params.EnumParam.DISPLAY_HLIST, + display=params.EnumParam.DISPLAY_COMBO, help='Adjust densities of sections individually:\n' '-Default: no adjustment performed\n' - '-Mode 1: sections fill the data range\n' - '-Mode 2: sections scaled to common mean and standard deviation.\n' - '-Mode 3: sections shifted to a common mean without scaling\n' - '-Mode 4: sections shifted to a common mean and then ' - 'rescale the resulting minimum and maximum densities ' - 'to the Min and Max values specified') + + '-Range between min and max: This option will scale the gray values' + 'to be in a range given by a minimum and a maximum values.' + 'This is the mode 1 in newstack flag -floatDensities.\n' + + '-Scaled to common mean and standard deviation: This is the most ' + 'common normalization procedure. The new tomogramss will have' + 'a mean and a standard deviation introduced by the user. Generaly,' + 'a zero mean and a standard deviation one is a good choice.' + 'This is the mode 2 in newstack flag -floatDensities.\n' + + '-Shifted to a common mean without scaling: This option only' + 'add an offset to the gray values of the tomograms. The offset will' + 'be calculated such as the new tomograms will present a mean gray value' + 'introduced by the user. This is the mode 3 in newstack flag ' + 'floatDensities.\n' + + '-shifted to mean and rescaled to a min and max: In this case, an ' + 'offset is added to the tomograms in order to achieve a mean gray value' + ' then they are rescale the resulting minimum and maximum densities ' + 'to the Min and Max values specified. This is the mode 4 in newstack' + ' flag -floatDensities.\n') + + groupMeanSd = form.addGroup('Mean and SD', + condition='floatDensities==2', + help='Scale all images to the given mean ' + 'and standard deviation. This option ' + 'implies -float 2 and is incompatible ' + 'with all other scaling options. If no ' + 'values are set, mean=0 and SD=1 by default') + + groupMeanSd.addParam('meanSdToggle', + params.EnumParam, + choices=['Yes', 'No'], + default=1, + label='Set mean and SD?', + display=params.EnumParam.DISPLAY_HLIST, + help='Set mean and SD values') + + groupMeanSd.addParam('scaleMean', + params.FloatParam, + default=0, + label='Mean', + help='Mean value for the rescaling') + + groupMeanSd.addParam('scaleSd', + params.FloatParam, + default=1, + label='SD', + help='Standard deviation value for the rescaling') + + groupScale = form.addGroup('Scaling values', + condition='floatDensities==4') + + groupScale.addParam('scaleMax', + params.FloatParam, + default=255, + label='Max.', + help='Maximum value for the rescaling') + + groupScale.addParam('scaleMin', + params.FloatParam, + default=0, + label='Min.', + help='Minimum value for the rescaling') + form.addParam('modeToOutput', params.EnumParam, - choices=['default', '4-bit', 'byte', 'signed 16-bit', + expertLevel=params.LEVEL_ADVANCED, + choices=['default', '4-bit', '8-bit', 'signed 16-bit', 'unsigned 16-bit', '32-bit float'], default=0, label='Storage data type', - display=params.EnumParam.DISPLAY_HLIST, - help='Apply one density scaling to all sections to map ' - 'current min and max to the given Min and Max. The ' - 'storage mode of the output file. The default is ' - 'the mode of the first input file, except for a ' - '4-bit input file, where the default is to output ' - 'as bytes') + display=params.EnumParam.DISPLAY_COMBO, + help='The storage mode of the output file. The ' + 'default is the mode of the first input file, ' + 'except for a 4-bit input file, where the default ' + 'is to output as bytes') form.addParam('scaleRangeToggle', params.EnumParam, choices=['Yes', 'No'], condition="floatDensities==0 or floatDensities==1 or floatDensities==3", - default=1, + default=0, label='Set scaling range values?', display=params.EnumParam.DISPLAY_HLIST, help='This option will rescale the densities of all ' @@ -123,13 +208,13 @@ def _defineParams(self, form): form.addParam('antialias', params.EnumParam, choices=['None', 'Blackman', 'Triangle', 'Mitchell', - 'Lanczos 2', 'Lanczos 3'], + 'Lanczos 2 lobes', 'Lanczos 3 lobes'], default=5, label='Antialias method:', - display=params.EnumParam.DISPLAY_HLIST, + display=params.EnumParam.DISPLAY_COMBO, help='Type of antialiasing filter to use when reducing images.\n' 'The available types of filters are:\n\n' - 'None\n' + 'None - Antialias will not be applied\n' 'Blackman - fast but not as good at antialiasing as slower filters\n' 'Triangle - fast but smooths more than Blackman\n' 'Mitchell - good at antialiasing, smooths a bit\n' @@ -141,49 +226,6 @@ def _defineParams(self, form): 'based on images of natural scenes where there are ' 'sharp edges.') - groupMeanSd = form.addGroup('Mean and SD', - condition='floatDensities==2', - help='Scale all images to the given mean ' - 'and standard deviation. This option ' - 'implies -float 2 and is incompatible ' - 'with all other scaling options. If no ' - 'values are set, mean=0 and SD=1 by default') - - groupMeanSd.addParam('meanSdToggle', - params.EnumParam, - choices=['Yes', 'No'], - default=1, - label='Set mean and SD?', - display=params.EnumParam.DISPLAY_HLIST, - help='Set mean and SD values') - - groupMeanSd.addParam('scaleMean', - params.FloatParam, - default=0, - label='Mean', - help='Mean value for the rescaling') - - groupMeanSd.addParam('scaleSd', - params.FloatParam, - default=1, - label='SD', - help='Standard deviation value for the rescaling') - - groupScale = form.addGroup('Scaling values', - condition='floatDensities==4') - - groupScale.addParam('scaleMax', - params.FloatParam, - default=255, - label='Max.', - help='Maximum value for the rescaling') - - groupScale.addParam('scaleMin', - params.FloatParam, - default=0, - label='Min.', - help='Minimum value for the rescaling') - form.addParam('processOddEven', params.BooleanParam, expertLevel=params.LEVEL_ADVANCED, diff --git a/imod/protocols/protocol_tomoProjection.py b/imod/protocols/protocol_tomoProjection.py index e1df8f76..0a01b71b 100644 --- a/imod/protocols/protocol_tomoProjection.py +++ b/imod/protocols/protocol_tomoProjection.py @@ -43,6 +43,14 @@ class ProtImodTomoProjection(ProtImodBase): Re-project a tomogram given a geometric description (axis and angles). More info: https://bio3d.colorado.edu/imod/doc/man/xyzproj.html + + This program will compute projections of a tomogram at a series of + tilts around either the X, the Y or the Z axis.\n + + A projection along a ray line is simply the average of the pixels in + the block along that line. However, rather than taking the values of + the pixels that lie near the ray, interpolation is used to sample den- + sity at points evenly spaced at one pixel intervals along the ray. """ _label = 'Tomo projection' @@ -59,25 +67,26 @@ def _defineParams(self, form): params.PointerParam, pointerClass='SetOfTomograms', important=True, - label='Input set of tomograms') + label='Input set of tomograms to be projected') + + line = form.addLine('Tilt angles (deg)', + help='Starting, ending, and increment tilt angle. Enter the same value for ' + 'starting and ending angle to get only one image') - form.addParam('minAngle', + line.addParam('minAngle', params.FloatParam, default=-60.0, - label='Minimum angle of rotation', - help='Minimum angle of the projection range') + label='Minimum rotation') - form.addParam('maxAngle', + line.addParam('maxAngle', params.FloatParam, default=60.0, - label='Maximum angle of rotation', - help='Maximum angle of the projection range') + label='Maximum rotation') - form.addParam('stepAngle', + line.addParam('stepAngle', params.FloatParam, default=2.0, - label='Step angle', - help='Step angle of the projection range') + label='Step angle') form.addParam('rotationAxis', params.EnumParam, diff --git a/imod/protocols/protocol_tomoReconstruction.py b/imod/protocols/protocol_tomoReconstruction.py index af84c07d..a1b2c60c 100644 --- a/imod/protocols/protocol_tomoReconstruction.py +++ b/imod/protocols/protocol_tomoReconstruction.py @@ -42,6 +42,23 @@ class ProtImodTomoReconstruction(ProtImodBase): More info: https://bio3d.colorado.edu/imod/doc/man/tilt.html + + It makes use of the IMOD tilt program. Tilt is a program for reconstructing + a three-dimensional object (tomogram) from a series of 2D projections. + The projections are assumed to arise from rotation about a fixed tilt axis, + subject to minor variations from this scheme.\n + + The program uses a number of different numerical strategies depending on + the complexity of the alignments needed to reconstruct the tomogram + and on whether the computation is being done by the central processing + unit (CPU) or the graphical processing unit (GPU). If there are no + local alignments being applied, then for processing on the CPU, the + program will do a preliminary stretching of each input line by the + cosine of the tilt angle. This stretching speeds up the direct back- + projection because each stretched input line is in register with the + lines of the output planes. When computing on the GPU, the program + does not use cosine stretching, thus avoiding the consequences of + interpolating the data twice.\n """ _label = 'Tomo reconstruction' @@ -55,64 +72,105 @@ def _defineParams(self, form): params.PointerParam, pointerClass='SetOfTiltSeries', important=True, - label='Input set of tilt-series') + label='Tilt Series') form.addParam('tomoThickness', params.FloatParam, default=1000, label='Tomogram thickness (voxels)', important=True, - help='Size in voxels of the tomogram in the z ' + help='Size in voxels of the tomogram along the z ' 'axis (beam direction).') - form.addParam('tomoShiftX', - params.FloatParam, - default=0, - label='Tomogram shift in X', - help='This entry allows one to shift the reconstructed ' - 'slice in X before it is output. If the X shift ' - 'is positive, the slice will be shifted to the ' - 'right, and the output will contain the left part ' - 'of the whole potentially reconstructable area.') form.addParam('tomoWidth', params.IntParam, default=0, label='Tomogram width (voxels)', help='Number of pixels to cut out in X, centered on the middle in X. Leave 0 for default X.') - form.addParam('tomoShiftZ', + + lineShift = form.addLine('Tomogram shift (Å)', + expertLevel=params.LEVEL_ADVANCED, + help="This entry allows one to shift the reconstructed" + " slice in X or Z before it is output. If the " + " X shift is positive, the slice will be shifted to " + " the right, and the output will contain the left " + " part of the whole potentially reconstructable area. " + " If the Z shift is positive, the slice is shifted " + " upward. The Z entry is optional and defaults to 0 when " + " omitted.") + + lineShift.addParam('tomoShiftX', params.FloatParam, default=0, - label='Tomogram shift in Z', - help='This entry allows one to shift the reconstructed ' - 'slice in Z before it is output. If the Z ' - 'shift is positive, the slice is shifted upward. ' - 'The Z entry is optional and defaults to 0 ' - 'when omitted.') + label=' in X ') + + lineShift.addParam('tomoShiftZ', + params.FloatParam, + default=0, + label=' in Z ') + + lineoffSet = form.addLine('Offset (deg) of the ', + expertLevel=params.LEVEL_ADVANCED, + help="* Tilt angle offset: pply an angle offset in degrees to all tilt " + "angles. This offset positively rotates the reconstructed sections anticlockwise.\n" + "* Tilt axis offset: Apply an offset to the tilt axis in a stack of full-sized " + "projection images, cutting the X-axis at NX/2 + offset instead of NX/2.") - form.addParam('angleOffset', + lineoffSet.addParam('angleOffset', params.FloatParam, default=0, - label='Angle offset', + label='Tilt angles ', help='Apply an angle offset in degrees to all tilt ' 'angles. This offset positively rotates the ' 'reconstructed sections anticlockwise.') - form.addParam('tiltAxisOffset', + lineoffSet.addParam('tiltAxisOffset', params.FloatParam, default=0, - label='Tilt axis offset', + label='Tilt axis', help='Apply an offset to the tilt axis in a stack of ' 'full-sized projection images, cutting the ' - 'X-axis at NX/2. + offset instead of NX/2. The ' + 'X-axis at NX/2 + offset instead of NX/2. The ' 'DELXX entry is optional and defaults to 0 ' 'when omitted.') + + + form.addParam('superSampleFactor', params.IntParam, default=2, label='Super-sampling factor', expertLevel=params.LEVEL_ADVANCED, - help='Compute slices in pixels smaller by this factor to reduce artifacts.') + help='Compute slices in pixels smaller by this factor to reduce artifacts.' + 'Super-sampling refers to computing the back projection in a slice ' + 'larger by an integer factor in each dimension, which is done here in ' + 'one of two ways: by interpolating the projection data at smaller ' + 'intervals during backprojection, or by expanding the input lines by ' + 'the factor using sync interpolation (padding in Fourier space). ' + 'Super-sampling will reduce the rays along the projection angles that ' + 'appear to reflect from the edges of a Fourier transform of an X/Z slice.' + 'These rays result from back-projecting into discrete pixels and represent' + 'inappropriate information generated by the transitions between successive ' + 'pixels along a backprojection ray. Super-sampling by 2 will remove most ' + 'of these rays, especially oblique ones. The additional benefit (amount ' + 'of change in the image) of going from 2 to 3 is about 10% as large as ' + 'that of super-sampling by 2; it is about 3% going from 3 to 4 and 1.5% ' + 'going from 4 to 8 (the highest allowed value). The super-sampled slice ' + 'is reduced to the original resolution by cropping its Fourier transform. ' + 'Super-sampling alone with the first method does not reduce the rays in ' + 'the corners of the Fourier transform past 0.5 cycle/pixel. These rays ' + 'are also inappropriate since they originate from lower-frequency ' + 'information in the projection images, so they are removed from the ' + 'cropped Fourier transform before inverting it. This removal has an ' + 'added benefit about 1/3 as large as the benefit from supersampling by 2. ' + 'The effect of these removals is a subtle change in the noise, and a ' + 'benefit may show up only with subvolume averaging.\n' + 'The additional effect of expanding the input lines is to avoid attenu-' + 'ating frequencies past half Nyquist (0.25/pixel) any more than is ' + 'achieved with the falloff of the radial filter. This will be noticeable ' + 'in a tomogram and would be particularly helpful if setting the radial ' + 'filter cutoff higher than the default for subvolume averaging.\n') form.addParam('fakeInteractionsSIRT', params.IntParam, @@ -125,31 +183,34 @@ def _defineParams(self, form): 'Gaussian filter is applied at the high-frequency ' 'end of the filter. The functioning of this filter ' 'is described in: \n\t' - 'https://bio3d.colorado.edu/imod/doc/man/tilt.html') + 'https://bio3d.colorado.edu/imod/doc/man/tilt.html' + 'This entry corresponds to the imod parameter – FakeSIRTiterations') groupRadialFrequencies = form.addGroup('Radial filtering', - help='This entry controls low-pass ' - 'filtering with the radial weighting ' - 'function. The radial weighting ' - 'function is linear away from the ' - 'origin out to the distance in ' - 'reciprocal space specified by the ' - 'first value, followed by a Gaussian ' - 'fall-off determined by the ' + help='This entry controls low-pass filtering with the radial weighting ' + 'function. The radial weighting function is linear away from the ' + 'origin out to the distance in reciprocal space specified by the ' + 'first value, followed by a Gaussian fall-off determined by the ' 'second value.', expertLevel=params.LEVEL_ADVANCED) groupRadialFrequencies.addParam('radialFirstParameter', params.FloatParam, default=0.35, - label='First parameter', - help='Linear region value') + label='Cutoff linear region', + help='Linear region of the filter. This is the first' + 'parameter of the -RADIAL flag in IMOD tilt command.' + 'If the cutoff is greater than 1 the distances are interpreted ' + 'as pixels in Fourier space; otherwise they are treated as ' + 'frequencies in cycles/pixel, which range from 0 to 0.5. ' + 'Use a cutoff of 0.5 for no low-pass filtering.') groupRadialFrequencies.addParam('radialSecondParameter', params.FloatParam, default=0.035, - label='Second parameter', - help='Gaussian fall-off parameter') + label='Radial fall-off', + help='Gaussian fall-off parameter. The sigma (or standard deviation) ' + 'of the Gaussian is the second value times 0.707.') form.addHidden(params.USE_GPU, params.BooleanParam, diff --git a/imod/protocols/protocol_tsNormalization.py b/imod/protocols/protocol_tsNormalization.py index cd390a9d..560b21e0 100644 --- a/imod/protocols/protocol_tsNormalization.py +++ b/imod/protocols/protocol_tsNormalization.py @@ -41,6 +41,24 @@ class ProtImodTSNormalization(ProtImodBase): Normalize input tilt-series and change its storing formatting. More info: https://bio3d.colorado.edu/imod/doc/man/newstack.html + + IMOD tilt series preprocess makes use of the Newstack command. + In particular, three functionalities are possible:\n + + _1 Binning_: The protocol also allows to bin tilt series. This + means to reduce the dimensions of the tilt series keeping but + keeping most of the information. The binning factor or simply + binning is an integer number and represent the scaling factor + of the images. Binning 2 means that the original images will + be twice the binned ones. + _2 Normalization_: This protocol allows to scale the gray values + of the images, also called normalization, to a common range or + mean of density. The most used normalization consists in zero + mean and standard deviation one.\n + + _3 storage format_: IMOD is able to modify the number of bit of + the stored data in order to reduce the disc occupancy. + """ _label = 'Tilt-series preprocess' @@ -61,9 +79,12 @@ def _defineParams(self, form): default=1, label='Binning', important=True, - help='Binning to be applied to the normalized tilt-series ' - 'in IMOD convention. Images will be binned by the ' - 'given factor. Must be an integer bigger than 1') + help='Binning is an scaling factor for the output images. ' + 'Must be an integer greater than 1. IMOD uses ordinary' + 'binning to reduce images in size by the given factor. ' + 'The value of a binned pixel is the average of pixel ' + 'values in each block of pixels being binned. Binning ' + 'is applied before all') form.addParam('applyAlignment', params.BooleanParam, @@ -73,29 +94,88 @@ def _defineParams(self, form): form.addParam('floatDensities', params.EnumParam, - choices=['default', '1', '2', '3', '4'], + choices=['No adjust', + 'range between min and max', + 'scaled to common mean and standard deviation', + 'shifted to a common mean without scaling', + 'shifted to mean and rescaled to a min and max'], default=0, label='Adjust densities mode', - display=params.EnumParam.DISPLAY_HLIST, + display=params.EnumParam.DISPLAY_COMBO, help='Adjust densities of sections individually:\n' '-Default: no adjustment performed\n' - '-Mode 1: sections fill the data range\n' - '-Mode 2: sections scaled to common mean and standard deviation.\n' - '-Mode 3: sections shifted to a common mean without scaling\n' - '-Mode 4: sections shifted to a common mean and then ' - 'rescale the resulting minimum and maximum densities ' - 'to the Min and Max values specified') + + '-Range between min and max: This option will scale the gray values' + 'to be in a range given by a minimum and a maximum values.' + 'This is the mode 1 in newstack flag -floatDensities.\n' + + '-Scaled to common mean and standard deviation: This is the most ' + 'common normalization procedure. The new tilt series will have' + 'a meand and a standard deviation introduced by the user. Generaly,' + 'a zero meand and a standard deviation one is a good choice.' + 'This is the mode 2 in newstack flag -floatDensities.\n' + + '-Shifted to a common mean without scaling: This option only' + 'add an offset to the gray values of the images. The offset will' + 'be calculated such as the new images will present a mean gray value' + 'introduced by the user. This is the mode 3 in newstack flag ' + 'floatDensities.\n' + + '-shifted to mean and rescaled to a min and max: In this case, an ' + 'offset is added to the images in order to achieve a mean gray value' + ' then they are rescale the resulting minimum and maximum densities ' + 'to the Min and Max values specified. This is the mode 4 in newstack' + ' flag -floatDensities.\n') + + groupMeanSd = form.addGroup('Mean and SD', + condition='floatDensities==2', + help='Scale all images to the given mean ' + 'and standard deviation.') + + groupMeanSd.addParam('meanSdToggle', + params.EnumParam, + choices=['Yes', 'No'], + default=1, + label='Set mean and SD?', + display=params.EnumParam.DISPLAY_HLIST, + help='Set mean and SD values') + + groupMeanSd.addParam('scaleMean', + params.FloatParam, + default=0, + label='Mean', + help='Mean value for the rescaling') + + groupMeanSd.addParam('scaleSd', + params.FloatParam, + default=1, + label='SD', + help='Standard deviation value for the rescaling') + + groupScale = form.addGroup('Scaling values', + condition='floatDensities==4') + + groupScale.addParam('scaleMax', + params.FloatParam, + default=255, + label='Max.', + help='Maximum value for the rescaling') + + groupScale.addParam('scaleMin', + params.FloatParam, + default=0, + label='Min.', + help='Minimum value for the rescaling') form.addParam('modeToOutput', params.EnumParam, - choices=['default', '4-bit', 'byte', 'signed 16-bit', + expertLevel=params.LEVEL_ADVANCED, + choices=['default', '4-bit', '8-bit', 'signed 16-bit', 'unsigned 16-bit', '32-bit float'], default=0, label='Storage data type', - display=params.EnumParam.DISPLAY_HLIST, - help='Apply one density scaling to all sections to ' - 'map current min and max to the given Min and ' - 'Max. The storage mode of the output file. The ' + display=params.EnumParam.DISPLAY_COMBO, + help='The storage mode of the output file. The ' 'default is the mode of the first input file, ' 'except for a 4-bit input file, where the default ' 'is to output as bytes') @@ -104,7 +184,7 @@ def _defineParams(self, form): params.EnumParam, choices=['Yes', 'No'], condition="floatDensities==0 or floatDensities==1 or floatDensities==3", - default=1, + default=0, label='Set scaling range values?', display=params.EnumParam.DISPLAY_HLIST, help='This option will rescale the densities of all ' @@ -128,14 +208,15 @@ def _defineParams(self, form): form.addParam('antialias', params.EnumParam, + expertLevel=params.LEVEL_ADVANCED, choices=['None', 'Blackman', 'Triangle', 'Mitchell', - 'Lanczos 2', 'Lanczos 3'], + 'Lanczos 2 lobes', 'Lanczos 3 lobes'], default=5, label='Antialias method:', - display=params.EnumParam.DISPLAY_HLIST, + display=params.EnumParam.DISPLAY_COMBO, help='Type of antialiasing filter to use when reducing images.\n' 'The available types of filters are:\n\n' - 'None\n' + 'None - Antialias will not be applied\n' 'Blackman - fast but not as good at antialiasing as slower filters\n' 'Triangle - fast but smooths more than Blackman\n' 'Mitchell - good at antialiasing, smooths a bit\n' @@ -147,49 +228,6 @@ def _defineParams(self, form): 'based on images of natural scenes where there are ' 'sharp edges.') - groupMeanSd = form.addGroup('Mean and SD', - condition='floatDensities==2', - help='Scale all images to the given mean ' - 'and standard deviation. This option ' - 'implies -float 2 and is incompatible ' - 'with all other scaling options. If no ' - 'values are set, mean=0 and SD=1 by default') - - groupMeanSd.addParam('meanSdToggle', - params.EnumParam, - choices=['Yes', 'No'], - default=1, - label='Set mean and SD?', - display=params.EnumParam.DISPLAY_HLIST, - help='Set mean and SD values') - - groupMeanSd.addParam('scaleMean', - params.FloatParam, - default=0, - label='Mean', - help='Mean value for the rescaling') - - groupMeanSd.addParam('scaleSd', - params.FloatParam, - default=1, - label='SD', - help='Standard deviation value for the rescaling') - - groupScale = form.addGroup('Scaling values', - condition='floatDensities==4') - - groupScale.addParam('scaleMax', - params.FloatParam, - default=255, - label='Max.', - help='Maximum value for the rescaling') - - groupScale.addParam('scaleMin', - params.FloatParam, - default=0, - label='Min.', - help='Minimum value for the rescaling') - form.addParam('processOddEven', params.BooleanParam, expertLevel=params.LEVEL_ADVANCED, diff --git a/imod/protocols/protocol_xCorrPrealignment.py b/imod/protocols/protocol_xCorrPrealignment.py index 5c29b065..3f50d26f 100644 --- a/imod/protocols/protocol_xCorrPrealignment.py +++ b/imod/protocols/protocol_xCorrPrealignment.py @@ -36,13 +36,24 @@ from .. import Plugin, utils from .protocol_base import ProtImodBase +from .protocol_forms import CommonIMODforms -class ProtImodXcorrPrealignment(ProtImodBase): +class ProtImodXcorrPrealignment(ProtImodBase, CommonIMODforms): """ Tilt-series cross correlation alignment based on the IMOD procedure. More info: https://bio3d.colorado.edu/imod/doc/man/tiltxcorr.html + + Tiltxcorr uses cross-correlation to find an initial translational + alignment between successive images of a tilt series. For a given pair + of images, it stretches the image with the larger tilt angle perpendic- + ular to the tilt axis, by an amount equal to the ratio of the cosines + of the two tilt angles (cosine stretch). The stretched image is corre- + lated with the other image, and the position of the peak of the corre- + lation indicates the relative shift between the images. + + """ _label = 'Coarse prealignment' @@ -57,7 +68,7 @@ def _defineParams(self, form): params.PointerParam, pointerClass='SetOfTiltSeries', important=True, - label='Input set of tilt-series') + label='Tilt-series to be prealigned') form.addParam('cumulativeCorr', params.EnumParam, @@ -65,7 +76,7 @@ def _defineParams(self, form): default=1, label='Use cumulative correlation?', display=params.EnumParam.DISPLAY_HLIST, - help='With this option, the program will take the image at zero tilt as the first' + help='The program will take the image at zero tilt as the first' 'reference, and correlate it with the image at the next most negative tilt.' 'It will then add the aligned image to the first reference to make the ' 'reference for the next tilt. At each tilt, the reference will be the sum of ' @@ -79,18 +90,25 @@ def _defineParams(self, form): label='Generate interpolated tilt-series?', important=True, display=params.EnumParam.DISPLAY_HLIST, - help='Generate and save the interpolated tilt-series ' - 'applying the obtained transformation matrices.') + help='Generate and save the interpolated tilt-series applying the obtained transformation ' + 'matrices.\n' + 'By default, the output of this protocol will be a tilseries that will have associated' + 'the alignment information as a transformation matrix. When this option is set as Yes, ' + 'then a second output, called interpolated tilt series, is generated. The interpolated tilt ' + 'series should be used for visualization purpose but not for image processing') form.addParam('binning', params.IntParam, condition='computeAlignment==0', default=1, label='Binning for the interpolated', - help='Binning to be applied to the interpolated ' - 'tilt-series in IMOD convention. Images will be ' - 'binned by the given factor. Must be an integer ' - 'bigger than 1') + help='Binning to be applied to the interpolated tilt-series in IMOD ' + 'convention. \n' + 'Binning is an scaling factor given by an integer greater than 1. ' + 'IMOD uses ordinary binning to reduce images in size by the given factor. ' + 'The value of a binned pixel is the average of pixel values in each block ' + 'of pixels being binned. Binning is applied before all other image ' + 'transformations.') form.addParam('Trimming parameters', params.LabelParam, label='Tilt axis angle detected from import. In case another value is desired please adjust the ' @@ -106,108 +124,10 @@ def _defineParams(self, form): 'Usually, it will be 90 degrees less than the RotationAngle in a ' 'system with no axis inversions') - trimming = form.addGroup('Trimming parameters', - expertLevel=params.LEVEL_ADVANCED) - - xtrimming = trimming.addLine('Horizontal: Number of pixels to avoid from the', - expertLevel=params.LEVEL_ADVANCED, - help="Starting and ending X coordinates of a region to correlate, " - "based on the position of the region at zero tilt.") - - xtrimming.addParam('xmin', - params.IntParam, - label='left', - allowsNull=True, - expertLevel=params.LEVEL_ADVANCED) - - xtrimming.addParam('xmax', - params.IntParam, - label='right', - allowsNull=True, - expertLevel=params.LEVEL_ADVANCED) - - ytrimming = trimming.addLine('Vertical: Number of pixels to avoid from the', - expertLevel=params.LEVEL_ADVANCED, - help="Starting and ending Y coordinates of a region to correlate.") - - ytrimming.addParam('ymin', - params.IntParam, - label='top', - allowsNull=True, - expertLevel=params.LEVEL_ADVANCED) - - ytrimming.addParam('ymax', - params.IntParam, - label='botton', - allowsNull=True, - expertLevel=params.LEVEL_ADVANCED) - - filtering = form.addGroup('Filtering parameters', - expertLevel=params.LEVEL_ADVANCED) - - line1 = filtering.addLine('High pass filter', - expertLevel=params.LEVEL_ADVANCED, - help="Some high pass filtering, using a small value of Sigma1 such " - "as 0.03, may be needed to keep the program from being misled by very " - "large scale features in the images. If the images are noisy, some low " - "pass filtering with Sigma2 and Radius2 is appropriate (e.g. 0.05 for " - " Sigma2, 0.25 for Radius2). If the images are binned, these values " - "specify frequencies in the binned image, so a higher cutoff (less filtering) " - "might be appropriate.\n\n" - "" - "*FilterRadius1*: Low spatial frequencies in the cross-correlation " - "will be attenuated by a Gaussian curve that is 1 " - "at this cutoff radius and falls off below this " - "radius with a standard deviation specified by " - "FilterSigma2. Spatial frequency units range from " - "0 to 0.5.\n" - "*Filter sigma 1*: Sigma value to filter low frequencies in the " - "correlations with a curve that is an inverted " - "Gaussian. This filter is 0 at 0 frequency and " - "decays up to 1 with the given sigma value. " - "However, if a negative value of radius1 is entered, " - "this filter will be zero from 0 to " - "|radius1| then decay up to 1.") - - line1.addParam('filterRadius1', - params.FloatParam, - label='Filter radius 1', - default='0.0', - expertLevel=params.LEVEL_ADVANCED) - - line1.addParam('filterSigma1', - params.FloatParam, - label='Filter sigma 1', - default='0.03', - expertLevel=params.LEVEL_ADVANCED) - - line2 = filtering.addLine('Low pass filter', - expertLevel=params.LEVEL_ADVANCED, - help="If the images are noisy, some low " - "pass filtering with Sigma2 and Radius2 is appropriate (e.g. 0.05 for " - " Sigma2, 0.25 for Radius2). If the images are binned, these values " - "specify frequencies in the binned image, so a higher cutoff (less filtering) " - "might be appropriate.\n\n" - "*Filter radius 2*: High spatial frequencies in the cross-correlation " - "will be attenuated by a Gaussian curve that is 1 " - "at this cutoff radius and falls off above this " - "radius with a standard deviation specified by " - "FilterSigma2.\n" - "*Filter sigma 2*: Sigma value for the Gaussian rolloff below and " - "above the cutoff frequencies specified by " - "FilterRadius1 and FilterRadius2") - - line2.addParam('filterRadius2', - params.FloatParam, - label='Filter radius 2', - default='0.25', - expertLevel=params.LEVEL_ADVANCED) - - line2.addParam('filterSigma2', - params.FloatParam, - label='Filter sigma 2', - default='0.05', - expertLevel=params.LEVEL_ADVANCED) + trimming = form.addGroup('Trimming parameters', expertLevel=params.LEVEL_ADVANCED) + + self.trimimgForm(trimming, pxTrimCondition='False', correlationCondition='True', levelType=params.LEVEL_ADVANCED) + self.filteringParametersForm(form, condition='True', levelType=params.LEVEL_ADVANCED) # -------------------------- INSERT steps functions ----------------------- def _insertAllSteps(self): diff --git a/imod/protocols/protocol_xRaysEraser.py b/imod/protocols/protocol_xRaysEraser.py index d8b032e6..bd1d4459 100644 --- a/imod/protocols/protocol_xRaysEraser.py +++ b/imod/protocols/protocol_xRaysEraser.py @@ -38,9 +38,77 @@ class ProtImodXraysEraser(ProtImodBase): """ - Erase X-rays from aligned tilt-series based on the IMOD procedure. + Erase Xrays from aligned tilt-series based on the IMOD procedure. More info: https://bio3d.colorado.edu/imod/doc/man/ccderaser.html + + This program replaces deviant pixels with interpolated values from + surrounding pixels. It is designed to correct defects in electron + microscope images from CCD cameras. It can use two algorithms to + automatically remove peaks in intensity caused by X-rays.\n + + The automatic removal of X-rays works by dividing the area of each + image into patches for scanning. The mean and standard deviation (SD) + of the pixels in a patch are computed. The patch is then scanned for + pixels that deviate from the mean by more than a criterion number of + SDs (the scan criterion, a relatively low number to keep from missing + peaks). When such a pixel is found, the program searches neighboring + pixels to find a peak in intensity. It then computes the mean and SD + of pixels in an annulus around the peak and makes sure that the peak + deviates from this local mean by more than a criterion number of SDs + (the peak criterion). Neighboring pixels inside the inner radius of + the annulus are added to the list of pixels to be replaced if they + deviate by a lower criterion (the grow criterion). The patch of pixels + is then replaced by fitting a polynomial to adjacent pixels and inter- + polating from the polynomial. If the peak does not deviate suffi- + ciently from this local mean, but is stronger than the mean of the scan + area by the scan criterion plus 1, then the mean and SD is again com- + puted in a larger annulus. If the peak deviates from this mean by a + number of SDs bigger than another criterion for extra-large peaks, a + patch of pixels is found, but it is replaced only if enough pixels dif- + fer from adjacent ones by large enough amounts (see the -big option + below). The reason for these two stages is that the inner radius for + the first stage must be set safely smaller than the radius of gold + beads to avoid erasing part of the beads, whereas the second stage can + work with larger areas because it has more stringent criteria that + reject gold beads. + + After the peaks are found in a scanning patch, the program next finds + the difference between each pixel and the mean of the eight adjacent + pixels. The mean and SD of this difference is computed, then pixels + are sought that deviate from the mean by yet another criterion, the + difference criterion. When such a pixel is found, neighboring pixels + are examined and added to the patch of pixels to replace if their dif- + ference exceeds the grow criterion. If the number of pixels in the + patch does not exceed a specified maximum, replacement proceeds as + above; otherwise the patch is ignored. + + Two methods are used because the first method is probably more reliable + for dealing with strong peaks that extend over several pixels, while + the second method is definitely better for finding small X-rays. + + After all the patches have been scanned for a section, the program then + searches for single pixels with large interpixel differences at the + edges of the image, over the width set by the -border option. A dif- + ference between a pixel and the mean of whatever adjacent pixels exist + is computed and its deviation from the overall mean interpixel differ- + ence is divided by the maximum SD of interpixel differences over all of + the scans. When this value exceeds the difference criterion and the + interpixel difference is greater than that of its neighbors, the pixel + is replaced with the mean. This procedure is iterated up to 4 times to + catch adjacent extreme pixels. + + Tuning the removal of X-rays would primarily involve adjusting two of + the criteria. The peak and difference criteria would be adjusted down + or up to increase or decrease the number of deviant pixels that are + found. The grow criterion could also be adjusted down or up depending + on whether too few or too many pixels are included in a patch that is + replaced, but this step is not usually done in practice. If there are + strong, large artifacts that are not being removed, the big difference + criterion for extra-large peaks should be lowered first, then if neces- + sary, the maximum radius and criterion strength for extra-large peaks + can be adjusted. + """ _label = 'X-rays eraser' @@ -56,24 +124,24 @@ def _defineParams(self, form): params.PointerParam, pointerClass='SetOfTiltSeries', important=True, - label='Input set of tilt-series') + label='Tilt Series') form.addParam('peakCriterion', params.FloatParam, - default=8.0, - label='Peak criterion', + default=10.0, + label='Peak criterion (in std)', expertLevel=params.LEVEL_ADVANCED, help='Criterion # of SDs above local mean for erasing ' - 'peak based on intensity (the default is 8 SDs)') + 'peak based on intensity (the default is 10 SDs)') form.addParam('diffCriterion', params.FloatParam, - default=6.0, - label='Difference criterion', + default=8.0, + label='Difference criterion (in std)', expertLevel=params.LEVEL_ADVANCED, help='Criterion # of SDs above mean pixel-to-pixel ' 'difference for erasing a peak based on ' - 'differences (the default is 6 SDs).') + 'differences (the default is 8 SDs).') form.addParam('maximumRadius', params.FloatParam, @@ -86,7 +154,7 @@ def _defineParams(self, form): form.addParam('bigDiffCriterion', params.IntParam, default=19, - label='Big difference criterion', + label='Big difference criterion (in std)', expertLevel=params.LEVEL_ADVANCED, help='An extra-large peak will be erased only if the ' 'value for the maximum difference between ' From edcf89f7d5e78f19e8706e82f81b5774a3bdcb76 Mon Sep 17 00:00:00 2001 From: Vilax Date: Tue, 16 Apr 2024 10:19:38 +0200 Subject: [PATCH 21/49] test passing --- imod/protocols/protocol_fiducialModel.py | 3 ++- imod/tests/test_protocols_imod.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/imod/protocols/protocol_fiducialModel.py b/imod/protocols/protocol_fiducialModel.py index 03eda9de..c8df1a47 100644 --- a/imod/protocols/protocol_fiducialModel.py +++ b/imod/protocols/protocol_fiducialModel.py @@ -79,7 +79,7 @@ def _patchTrackingForm(self, form, condition, levelType=params.LEVEL_NORMAL): patchtrack.addParam('sizeOfPatches', params.NumericListParam, label='Size of the patches (X,Y)', - allowsNull=True, + default='100 100', expertLevel=levelType, help="Size of the patches to track by correlation. In imod documentation " "(tiltxcorr: SizeOfPatchesXandY)") @@ -110,6 +110,7 @@ def _patchTrackingForm(self, form, condition, levelType=params.LEVEL_NORMAL): patchtrack.addParam('iterationsSubpixel', params.IntParam, + default=1, label='Iterations to increase subpixel accuracy', help="Number of iteration of each correlation to reduce interpolation of the peak position" "In imod documentation: (tiltxcorr: IterateCorrelations)") diff --git a/imod/tests/test_protocols_imod.py b/imod/tests/test_protocols_imod.py index ad94f3e5..c24bacc5 100644 --- a/imod/tests/test_protocols_imod.py +++ b/imod/tests/test_protocols_imod.py @@ -138,10 +138,11 @@ def _runXcorrPrealignment(cls, inputSoTS, computeAlignmentToggle, return cls.protXcorr @classmethod - def _runFiducialModels(cls, inputSoTS, twoSurfaces, fiducialRadius, + def _runFiducialModels(cls, inputSoTS, twoSurfaces, fiducialRadius, numberFiducial, rotationAngle, shiftsNearZeroFraction) -> ProtImodFiducialModel: cls.protFiducialAlignment = cls.newProtocol(ProtImodFiducialModel, + typeOfModel=0, inputSetOfTiltSeries=inputSoTS, twoSurfaces=twoSurfaces, fiducialRadius=fiducialRadius, From 76fc2361cc2743952b0a91970bad259253d44686 Mon Sep 17 00:00:00 2001 From: Vilax Date: Tue, 16 Apr 2024 11:01:22 +0200 Subject: [PATCH 22/49] =?UTF-8?q?adding=20PT=20test=20=C3=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- imod/tests/test_protocols_imod.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/imod/tests/test_protocols_imod.py b/imod/tests/test_protocols_imod.py index c24bacc5..c11ca001 100644 --- a/imod/tests/test_protocols_imod.py +++ b/imod/tests/test_protocols_imod.py @@ -138,7 +138,7 @@ def _runXcorrPrealignment(cls, inputSoTS, computeAlignmentToggle, return cls.protXcorr @classmethod - def _runFiducialModels(cls, inputSoTS, twoSurfaces, fiducialRadius, + def _runFiducialModels(cls, inputSoTS, twoSurfaces, fiducialRadius, numberFiducial, rotationAngle, shiftsNearZeroFraction) -> ProtImodFiducialModel: cls.protFiducialAlignment = cls.newProtocol(ProtImodFiducialModel, @@ -152,6 +152,14 @@ def _runFiducialModels(cls, inputSoTS, twoSurfaces, fiducialRadius, cls.launchProtocol(cls.protFiducialAlignment) return cls.protFiducialAlignment + @classmethod + def _runFiducialModelsPT(cls, inputSoTS) -> ProtImodFiducialModel: + cls.protFiducialAlignment = cls.newProtocol(ProtImodFiducialModel, + typeOfModel=1, + inputSetOfTiltSeries=inputSoTS) + cls.launchProtocol(cls.protFiducialAlignment) + return cls.protFiducialAlignment + @classmethod def _runFiducialAlignemnt(cls, inputSoLM, twoSurfaces, rotationAngle, computeAlignment, binning) -> ProtImodFiducialAlignment: @@ -383,6 +391,8 @@ def setUpClass(cls): rotationAngle=-12.5, shiftsNearZeroFraction=0.2) + cls.protFiducialModelsPT = cls._runFiducialModelsPT(inputSoTS=cls.protXcorr.TiltSeries) + cls.protFiducialAlignment = cls._runFiducialAlignemnt(inputSoLM=cls.protFiducialModels.FiducialModelGaps, twoSurfaces=0, rotationAngle=-12.5, From da355eb0ffe7307d58a6e96d3b7ea422280ff6cc Mon Sep 17 00:00:00 2001 From: JorMaister Date: Tue, 16 Apr 2024 17:17:22 +0200 Subject: [PATCH 23/49] Bump verion, update changes, move protocol form to base --- CHANGES.txt | 7 + imod/__init__.py | 2 +- imod/protocols/protocol_base.py | 133 ++++++++++++++++- imod/protocols/protocol_fiducialModel.py | 3 +- imod/protocols/protocol_forms.py | 149 ------------------- imod/protocols/protocol_xCorrPrealignment.py | 3 +- 6 files changed, 138 insertions(+), 159 deletions(-) delete mode 100644 imod/protocols/protocol_forms.py diff --git a/CHANGES.txt b/CHANGES.txt index 6fa2053f..f87a342f 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,3 +1,10 @@ +3.5.0: + Users: + - Protocols' help added. + - Help of the parameters was enhanced. + - New feature: patch tracking alignment added. + - Trimming options in fiducial model added + - The order of the parameters in the forms was slightly modified according to the user actions. 3.4.2: Developers: - Update the acquisition order in the CTFTomo objects (field added to that class in scipion-em-tomo v3.7.0). diff --git a/imod/__init__.py b/imod/__init__.py index 5b67fb2d..1ee66a4f 100644 --- a/imod/__init__.py +++ b/imod/__init__.py @@ -35,7 +35,7 @@ from .constants import IMOD_HOME, ETOMO_CMD, DEFAULT_VERSION, VERSIONS, IMOD_VIEWER_BINNING -__version__ = '3.4.2' +__version__ = '3.5.0' _logo = "icon.png" _references = ['Kremer1996', 'Mastronarde2017'] diff --git a/imod/protocols/protocol_base.py b/imod/protocols/protocol_base.py index b5e4b922..350b1b88 100644 --- a/imod/protocols/protocol_base.py +++ b/imod/protocols/protocol_base.py @@ -27,7 +27,7 @@ import os from pyworkflow.object import Set, CsvList, Pointer -from pyworkflow.protocol import STEPS_PARALLEL +from pyworkflow.protocol import STEPS_PARALLEL, params from pyworkflow.utils import path from pwem.emlib.image import ImageHandler from pwem.protocols import EMProtocol @@ -73,6 +73,132 @@ def __init__(self, **args): ProtTomoImportFiles.__init__(self, **args) + # -------------------------- DEFINE param functions ----------------------- + def _defineImportParams(self, form): + """ Method to define import params in protocol form """ + ProtTomoImportFiles._defineImportParams(self, form) + + @staticmethod + def trimimgForm(form, pxTrimCondition='False', correlationCondition='True', levelType=params.LEVEL_ADVANCED): + """ + Generally, this form will be integrated in a groupForm, the group form argument is form. A set of flags + control what elements are shown + """ + form.addParam('pxTrim', + params.NumericListParam, + condition=pxTrimCondition, + label='Pixels to trim (x y without coma separator)', + default="0 0", + help='Pixels to trim off each side in X and Y.\n' + 'Some trimming should be used for patch tracking', + expertLevel=levelType) + + xtrimming = form.addLine('Pixels to do correlation along X-axis', + expertLevel=levelType, + condition=correlationCondition, + help="Starting and ending X coordinates of a region to correlate, " + "based on the position of the region at zero tilt.") + + xtrimming.addParam('xmin', + params.IntParam, + label='X axis min (left)', + allowsNull=True, + expertLevel=levelType) + + xtrimming.addParam('xmax', + params.IntParam, + label='X axis max (right)', + allowsNull=True, + expertLevel=levelType) + + ytrimming = form.addLine('Pixels to do correlation along Y-axis', + expertLevel=levelType, + condition=correlationCondition, + help="Starting and ending Y coordinates of a region to correlate, " + "based on the position of the region at zero tilt.") + + ytrimming.addParam('ymin', + params.IntParam, + label='Y axis min (top)', + allowsNull=True, + expertLevel=levelType) + + ytrimming.addParam('ymax', + params.IntParam, + label='Y axis max (botton)', + allowsNull=True, + expertLevel=levelType) + + @staticmethod + def filteringParametersForm(form, condition, levelType=params.LEVEL_NORMAL): + filtering = form.addGroup('Filtering parameters', + condition=condition, + expertLevel=levelType) + + line1 = filtering.addLine('High pass filter', + expertLevel=levelType, + help="Some high pass filtering, using a small value of Sigma1 such " + "as 0.03, may be needed to keep the program from being misled by very " + "large scale features in the images. If the images are noisy, some low " + "pass filtering with Sigma2 and Radius2 is appropriate (e.g. 0.05 for " + " Sigma2, 0.25 for Radius2). If the images are binned, these values " + "specify frequencies in the binned image, so a higher cutoff (less filtering) " + "might be appropriate.\n\n" + "" + "*FilterRadius1*: Low spatial frequencies in the cross-correlation " + "will be attenuated by a Gaussian curve that is 1 " + "at this cutoff radius and falls off below this " + "radius with a standard deviation specified by " + "FilterSigma2. Spatial frequency units range from " + "0 to 0.5.\n" + "*Filter sigma 1*: Sigma value to filter low frequencies in the " + "correlations with a curve that is an inverted " + "Gaussian. This filter is 0 at 0 frequency and " + "decays up to 1 with the given sigma value. " + "However, if a negative value of radius1 is entered, " + "this filter will be zero from 0 to " + "|radius1| then decay up to 1.") + + line1.addParam('filterRadius1', + params.FloatParam, + label='Filter radius 1', + default='0.0', + expertLevel=levelType) + + line1.addParam('filterSigma1', + params.FloatParam, + label='Filter sigma 1', + default='0.03', + expertLevel=levelType) + + line2 = filtering.addLine('Low pass filter', + expertLevel=levelType, + help="If the images are noisy, some low " + "pass filtering with Sigma2 and Radius2 is appropriate (e.g. 0.05 for " + " Sigma2, 0.25 for Radius2). If the images are binned, these values " + "specify frequencies in the binned image, so a higher cutoff (less filtering) " + "might be appropriate.\n\n" + "*Filter radius 2*: High spatial frequencies in the cross-correlation " + "will be attenuated by a Gaussian curve that is 1 " + "at this cutoff radius and falls off above this " + "radius with a standard deviation specified by " + "FilterSigma2.\n" + "*Filter sigma 2*: Sigma value for the Gaussian rolloff below and " + "above the cutoff frequencies specified by " + "FilterRadius1 and FilterRadius2") + + line2.addParam('filterRadius2', + params.FloatParam, + label='Filter radius 2', + default='0.25', + expertLevel=levelType) + + line2.addParam('filterSigma2', + params.FloatParam, + label='Filter sigma 2', + default='0.05', + expertLevel=levelType) + @classmethod def worksInStreaming(cls): """ So far none of them work in streaming. Since this inherits from the import they were considered as "streamers". """ @@ -82,10 +208,6 @@ def defineExecutionPararell(self): self.stepsExecutionMode = STEPS_PARALLEL - def _defineImportParams(self, form): - """ Method to define import params in protocol form """ - ProtTomoImportFiles._defineImportParams(self, form) - # --------------------------- CALCULUS functions --------------------------- def tryExceptDecorator(func): """ This decorator wraps the step in a try/except module which adds @@ -100,6 +222,7 @@ def wrapper(self, tsId, *args): self._failedTs.append(tsId) return wrapper + def getTmpTSFile(self, tsId, tmpPrefix=None, suffix=".mrcs"): if tmpPrefix is None: tmpPrefix = self._getTmpPath(tsId) diff --git a/imod/protocols/protocol_fiducialModel.py b/imod/protocols/protocol_fiducialModel.py index c8df1a47..71ade3a2 100644 --- a/imod/protocols/protocol_fiducialModel.py +++ b/imod/protocols/protocol_fiducialModel.py @@ -34,10 +34,9 @@ from .. import Plugin, utils from .protocol_base import ProtImodBase -from .protocol_forms import CommonIMODforms -class ProtImodFiducialModel(ProtImodBase, CommonIMODforms): +class ProtImodFiducialModel(ProtImodBase): """ Construction of a fiducial model and alignment of tilt-series based on the IMOD procedure. diff --git a/imod/protocols/protocol_forms.py b/imod/protocols/protocol_forms.py deleted file mode 100644 index 5960923b..00000000 --- a/imod/protocols/protocol_forms.py +++ /dev/null @@ -1,149 +0,0 @@ -# ***************************************************************************** -# * -# * Authors: Federico P. de Isidro Gomez (fp.deisidro@cnb.csic.es) [1] -# * -# * [1] Centro Nacional de Biotecnologia, CSIC, Spain -# * -# * This program is free software; you can redistribute it and/or modify -# * it under the terms of the GNU General Public License as published by -# * the Free Software Foundation; either version 3 of the License, or -# * (at your option) any later version. -# * -# * This program is distributed in the hope that it will be useful, -# * but WITHOUT ANY WARRANTY; without even the implied warranty of -# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# * GNU General Public License for more details. -# * -# * You should have received a copy of the GNU General Public License -# * along with this program; if not, write to the Free Software -# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA -# * 02111-1307 USA -# * -# * All comments concerning this program package may be sent to the -# * e-mail address 'scipion@cnb.csic.es' -# * -# ***************************************************************************** - -import pyworkflow.protocol.params as params - - -class CommonIMODforms: - def trimimgForm(self, form, pxTrimCondition='False', correlationCondition='True', levelType=params.LEVEL_ADVANCED): - ''' - Generally, this form will be integrated in a groupForm, the group form argument is form. A set of flags - control what elements are shown - ''' - form.addParam('pxTrim', - params.NumericListParam, - condition=pxTrimCondition, - label='Pixels to trim (x y without coma separator)', - default="0 0", - help='Pixels to trim off each side in X and Y.\n' - 'Some trimming should be used for patch tracking', - expertLevel=levelType) - - xtrimming = form.addLine('Pixels to do correlation along X-axis', - expertLevel=levelType, - condition=correlationCondition, - help="Starting and ending X coordinates of a region to correlate, " - "based on the position of the region at zero tilt.") - - xtrimming.addParam('xmin', - params.IntParam, - label='X axis min (left)', - allowsNull=True, - expertLevel=levelType) - - xtrimming.addParam('xmax', - params.IntParam, - label='X axis max (right)', - allowsNull=True, - expertLevel=levelType) - - ytrimming = form.addLine('Pixels to do correlation along Y-axis', - expertLevel=levelType, - condition=correlationCondition, - help="Starting and ending Y coordinates of a region to correlate, " - "based on the position of the region at zero tilt.") - - ytrimming.addParam('ymin', - params.IntParam, - label='Y axis min (top)', - allowsNull=True, - expertLevel=levelType) - - ytrimming.addParam('ymax', - params.IntParam, - label='Y axis max (botton)', - allowsNull=True, - expertLevel=levelType) - - - def filteringParametersForm(self, form, condition, levelType=params.LEVEL_NORMAL): - filtering = form.addGroup('Filtering parameters', - condition=condition, - expertLevel=levelType) - - line1 = filtering.addLine('High pass filter', - expertLevel=levelType, - help="Some high pass filtering, using a small value of Sigma1 such " - "as 0.03, may be needed to keep the program from being misled by very " - "large scale features in the images. If the images are noisy, some low " - "pass filtering with Sigma2 and Radius2 is appropriate (e.g. 0.05 for " - " Sigma2, 0.25 for Radius2). If the images are binned, these values " - "specify frequencies in the binned image, so a higher cutoff (less filtering) " - "might be appropriate.\n\n" - "" - "*FilterRadius1*: Low spatial frequencies in the cross-correlation " - "will be attenuated by a Gaussian curve that is 1 " - "at this cutoff radius and falls off below this " - "radius with a standard deviation specified by " - "FilterSigma2. Spatial frequency units range from " - "0 to 0.5.\n" - "*Filter sigma 1*: Sigma value to filter low frequencies in the " - "correlations with a curve that is an inverted " - "Gaussian. This filter is 0 at 0 frequency and " - "decays up to 1 with the given sigma value. " - "However, if a negative value of radius1 is entered, " - "this filter will be zero from 0 to " - "|radius1| then decay up to 1.") - - line1.addParam('filterRadius1', - params.FloatParam, - label='Filter radius 1', - default='0.0', - expertLevel=levelType) - - line1.addParam('filterSigma1', - params.FloatParam, - label='Filter sigma 1', - default='0.03', - expertLevel=levelType) - - line2 = filtering.addLine('Low pass filter', - expertLevel=levelType, - help="If the images are noisy, some low " - "pass filtering with Sigma2 and Radius2 is appropriate (e.g. 0.05 for " - " Sigma2, 0.25 for Radius2). If the images are binned, these values " - "specify frequencies in the binned image, so a higher cutoff (less filtering) " - "might be appropriate.\n\n" - "*Filter radius 2*: High spatial frequencies in the cross-correlation " - "will be attenuated by a Gaussian curve that is 1 " - "at this cutoff radius and falls off above this " - "radius with a standard deviation specified by " - "FilterSigma2.\n" - "*Filter sigma 2*: Sigma value for the Gaussian rolloff below and " - "above the cutoff frequencies specified by " - "FilterRadius1 and FilterRadius2") - - line2.addParam('filterRadius2', - params.FloatParam, - label='Filter radius 2', - default='0.25', - expertLevel=levelType) - - line2.addParam('filterSigma2', - params.FloatParam, - label='Filter sigma 2', - default='0.05', - expertLevel=levelType) diff --git a/imod/protocols/protocol_xCorrPrealignment.py b/imod/protocols/protocol_xCorrPrealignment.py index 3f50d26f..3f2d7554 100644 --- a/imod/protocols/protocol_xCorrPrealignment.py +++ b/imod/protocols/protocol_xCorrPrealignment.py @@ -36,10 +36,9 @@ from .. import Plugin, utils from .protocol_base import ProtImodBase -from .protocol_forms import CommonIMODforms -class ProtImodXcorrPrealignment(ProtImodBase, CommonIMODforms): +class ProtImodXcorrPrealignment(ProtImodBase): """ Tilt-series cross correlation alignment based on the IMOD procedure. More info: From fffb4e46e3ebeb0a14d47ba958b831ed5710f40f Mon Sep 17 00:00:00 2001 From: JorMaister Date: Tue, 16 Apr 2024 17:19:47 +0200 Subject: [PATCH 24/49] fix typo --- imod/protocols/protocol_applyTransformationMatrix.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imod/protocols/protocol_applyTransformationMatrix.py b/imod/protocols/protocol_applyTransformationMatrix.py index ed2a1fe0..f8a63d10 100644 --- a/imod/protocols/protocol_applyTransformationMatrix.py +++ b/imod/protocols/protocol_applyTransformationMatrix.py @@ -61,7 +61,7 @@ def _defineParams(self, form): params.PointerParam, pointerClass='SetOfTiltSeries', important=True, - label='Input set of tilt-series to applied the transformation matrix') + label='Tilt-series to apply the transformation matrix') form.addParam('binning', params.IntParam, default=1, From df22b40c2c7a2efd845c7984d7bc7b5a4ff37b57 Mon Sep 17 00:00:00 2001 From: JorMaister Date: Wed, 17 Apr 2024 10:20:51 +0200 Subject: [PATCH 25/49] Undo some protocol renaming to avoid other plugin's test fail when calling them using the old name --- imod/protocols/__init__.py | 4 ++-- imod/protocols/protocol_tomoPreprocess.py | 8 ++------ imod/protocols/protocol_tsPreprocess.py | 2 +- imod/tests/test_protocols_imod.py | 6 +++--- 4 files changed, 8 insertions(+), 12 deletions(-) diff --git a/imod/protocols/__init__.py b/imod/protocols/__init__.py index d8869b18..386d5aac 100644 --- a/imod/protocols/__init__.py +++ b/imod/protocols/__init__.py @@ -40,10 +40,10 @@ from .protocol_fiducialAlignment import ProtImodFiducialAlignment from .protocol_fiducialModel import ProtImodFiducialModel from .protocol_goldBeadPicker3d import ProtImodGoldBeadPicker3d -from .protocol_tomoPreprocess import ProtImodTomoPreProcess +from .protocol_tomoPreprocess import ProtImodTomoNormalization from .protocol_tomoProjection import ProtImodTomoProjection from .protocol_tomoReconstruction import ProtImodTomoReconstruction -from .protocol_tsPreprocess import ProtImodTsPreprocess +from .protocol_tsPreprocess import ProtImodTsNormalization from .protocol_xCorrPrealignment import ProtImodXcorrPrealignment from .protocol_xRaysEraser import ProtImodXraysEraser diff --git a/imod/protocols/protocol_tomoPreprocess.py b/imod/protocols/protocol_tomoPreprocess.py index 9d402005..489138f7 100644 --- a/imod/protocols/protocol_tomoPreprocess.py +++ b/imod/protocols/protocol_tomoPreprocess.py @@ -23,20 +23,16 @@ # * e-mail address 'scipion@cnb.csic.es' # * # ***************************************************************************** - -import os - from pyworkflow import BETA from pyworkflow.object import Set import pyworkflow.protocol.params as params import pyworkflow.utils.path as pwpath from tomo.objects import Tomogram, SetOfTomograms - from .. import Plugin -from .protocol_base import ProtImodBase, EXT_MRC_ODD_NAME, EXT_MRC_EVEN_NAME, OUTPUT_TOMOGRAMS_NAME, MRC_EXT, ODD, EVEN +from .protocol_base import ProtImodBase,OUTPUT_TOMOGRAMS_NAME, MRC_EXT, ODD, EVEN -class ProtImodTomoPreProcess(ProtImodBase): +class ProtImodTomoNormalization(ProtImodBase): """ Normalize input tomogram and change its storing formatting. More info: diff --git a/imod/protocols/protocol_tsPreprocess.py b/imod/protocols/protocol_tsPreprocess.py index ab425d13..4c1ddf45 100644 --- a/imod/protocols/protocol_tsPreprocess.py +++ b/imod/protocols/protocol_tsPreprocess.py @@ -33,7 +33,7 @@ from ..utils import genXfFile -class ProtImodTsPreprocess(ProtImodBase): +class ProtImodTsNormalization(ProtImodBase): """ Normalize input tilt-series and change its storing formatting. More info: diff --git a/imod/tests/test_protocols_imod.py b/imod/tests/test_protocols_imod.py index 15ab217a..800cc237 100644 --- a/imod/tests/test_protocols_imod.py +++ b/imod/tests/test_protocols_imod.py @@ -111,7 +111,7 @@ def _runTSNormalization(cls, inputSoTS, binning, floatDensities, modeToOutput, scaleRangeToggle, scaleRangeMax, scaleRangeMin, meanSdToggle, scaleMean, scaleSd, scaleMax, scaleMin): - cls.protTSNormalization = cls.newProtocol(ProtImodTsPreprocess, + cls.protTSNormalization = cls.newProtocol(ProtImodTsNormalization, inputSetOfTiltSeries=inputSoTS, binning=binning, floatDensities=floatDensities, @@ -197,9 +197,9 @@ def _runTomoNormalization(cls, inputSetOfTomograms, binning, floatDensities, modeToOutput, scaleRangeToggle, scaleRangeMax, scaleRangeMin, meanSdToggle, - scaleMean, scaleSd, scaleMax, scaleMin) -> ProtImodTomoPreProcess: + scaleMean, scaleSd, scaleMax, scaleMin) -> ProtImodTomoNormalization: - cls.protTomoNormalization = cls.newProtocol(ProtImodTomoPreProcess, + cls.protTomoNormalization = cls.newProtocol(ProtImodTomoNormalization, inputSetOfTomograms=inputSetOfTomograms, binning=binning, floatDensities=floatDensities, From be77cd1c6101a1de462cd077475646821bbe2b0a Mon Sep 17 00:00:00 2001 From: JorMaister Date: Fri, 19 Apr 2024 11:05:22 +0200 Subject: [PATCH 26/49] Adapt the patch tracking part of the fiducial model protocol to the new refactorization --- imod/protocols/protocol_fiducialModel.py | 236 ++++++++++------------- 1 file changed, 104 insertions(+), 132 deletions(-) diff --git a/imod/protocols/protocol_fiducialModel.py b/imod/protocols/protocol_fiducialModel.py index e74b6393..0c164718 100644 --- a/imod/protocols/protocol_fiducialModel.py +++ b/imod/protocols/protocol_fiducialModel.py @@ -114,7 +114,8 @@ def _patchTrackingForm(self, form, condition, levelType=params.LEVEL_NORMAL): help="Number of iteration of each correlation to reduce interpolation of the peak position" "In imod documentation: (tiltxcorr: IterateCorrelations)") - self.trimimgForm(patchtrack, pxTrimCondition='True', correlationCondition='True',levelType=params.LEVEL_ADVANCED) + self.trimimgForm(patchtrack, pxTrimCondition='True', correlationCondition='True', + levelType=params.LEVEL_ADVANCED) self.filteringParametersForm(form, condition=condition, levelType=params.LEVEL_ADVANCED) def _fiducialSeedForm(self, form, condition, levelType=params.LEVEL_NORMAL): @@ -199,34 +200,19 @@ def _insertAllSteps(self): self._initialize() for tsId in self.tsDict.keys(): self._insertFunctionStep(self.convertInputStep, tsId) - self._insertFunctionStep(self.generateTrackComStep, tsId) - self._insertFunctionStep(self.generateFiducialSeedStep, tsId) - self._insertFunctionStep(self.generateFiducialModelStep, tsId) + if self.typeOfModel == self.FIDUCIAL_MODEL: + self._insertFunctionStep(self.generateTrackComStep, tsId) + self._insertFunctionStep(self.generateFiducialSeedStep, tsId) + self._insertFunctionStep(self.generateFiducialModelStep, tsId) + else: + self._insertFunctionStep(self.xcorrStep, tsId) + self._insertFunctionStep(self.chopcontsStep, tsId) self._insertFunctionStep(self.translateFiducialPointModelStep, tsId) self._insertFunctionStep(self.computeOutputModelsStep, tsId) self._insertFunctionStep(self.createOutputFailedSetStep, tsId) self._insertFunctionStep(self.createOutputStep) - # VILAS - # self._failedTs = [] - # - # for ts in self.inputSetOfTiltSeries.get(): - # tsObjId = ts.getObjId() - # self._insertFunctionStep(self.convertInputStep, tsObjId) - # if self.typeOfModel == self.FIDUCIAL_MODEL: - # self._insertFunctionStep(self.generateTrackComStep, tsObjId) - # self._insertFunctionStep(self.generateFiducialSeedStep, tsObjId) - # self._insertFunctionStep(self.generateFiducialModelStep, tsObjId) - # else: - # self._insertFunctionStep(self.xcorrStep, tsObjId) - # self._insertFunctionStep(self.chopcontsStep, tsObjId) - # self._insertFunctionStep(self.translateFiducialPointModelStep, tsObjId) - # self._insertFunctionStep(self.computeOutputModelsStep, tsObjId) - # self._insertFunctionStep(self.createOutputFailedSet, tsObjId) - # - # self._insertFunctionStep(self.createOutputStep) - # --------------------------- STEPS functions ----------------------------- def _initialize(self): self._failedTs = [] @@ -442,115 +428,101 @@ def computeOutputModelsStep(self, tsId): output.update(landmarkModelGaps) output.write() - # VILAS - # def xcorrStep(self, tsObjId): - # ''' - # Imod uses the next command line for the xcorr alignment - # $tiltxcorr -StandardInput - # InputFile cryo_preali.mrc - # OutputFile cryo_pt.fid - # RotationAngle -12.6 - # TiltFile cryo.rawtlt - # FilterRadius2 0.125 - # FilterSigma1 0.03 - # FilterSigma2 0.03 - # BordersInXandY 102,102 - # IterateCorrelations 1 - # SizeOfPatchesXandY 680,680 - # OverlapOfPatchesXandY 0.33,0.33 - # PrealignmentTransformFile cryo.prexg - # ImagesAreBinned 1 - # ''' - # ts = self._getTiltSeries(tsObjId) - # tsId = ts.getTsId() - # extraPrefix = self._getExtraPath(tsId) - # tmpPrefix = self._getTmpPath(tsId) - # - # firstItem = ts.getFirstItem() - # angleFilePath = os.path.join(tmpPrefix, firstItem.parseFileName(extension=".tlt")) - # - # xfFile = os.path.join(tmpPrefix, firstItem.parseFileName(extension=".xf")) - # ts.writeXfFile(xfFile) - # - # borders = self.pxTrim.getListFromValues() - # sizePatches = self.sizeOfPatches.getListFromValues() - # - # - # BordersInXandY = '%d,%d' % (borders[0], borders[1]) - # SizeOfPatchesXandY = '%d,%d' % (sizePatches[0], sizePatches[1]) - # - # paramsTiltXCorr = { - # 'inputFile': os.path.join(tmpPrefix, firstItem.parseFileName()), - # 'outputFile': os.path.join(extraPrefix, firstItem.parseFileName(suffix="_pt", extension=".fid")), - # 'RotationAngle': ts.getAcquisition().getTiltAxisAngle(), - # 'TiltFile': angleFilePath, - # 'FilterRadius2': self.filterRadius2.get(), - # 'FilterSigma1': self.filterSigma1.get(), - # 'FilterSigma2': self.filterSigma2.get(), - # 'BordersInXandY': BordersInXandY, - # 'IterateCorrelations': self.iterationsSubpixel.get(), - # 'SizeOfPatchesXandY': SizeOfPatchesXandY, - # 'PrealignmentTransformFile': xfFile, - # - # 'ImagesAreBinned': 1, - # } - # argsTiltXCorr = " " \ - # "-InputFile %(inputFile)s " \ - # "-OutputFile %(outputFile)s " \ - # "-RotationAngle %(RotationAngle)s " \ - # "-TiltFile %(TiltFile)s " \ - # "-FilterRadius2 %(FilterRadius2)s " \ - # "-FilterSigma1 %(FilterSigma1)s " \ - # "-FilterSigma2 %(FilterSigma2)s " \ - # "-BordersInXandY %(BordersInXandY)s " \ - # "-IterateCorrelations %(IterateCorrelations)s " \ - # "-SizeOfPatchesXandY %(SizeOfPatchesXandY)s " \ - # "-PrealignmentTransformFile %(PrealignmentTransformFile)s " \ - # "-ImagesAreBinned %(ImagesAreBinned)s " - # - # if self.patchLayout.get() == 0: - # patchesXY = self.overlapPatches.getListFromValues(caster=float) - # OverlapOfPatchesXandY = '%f,%f' % (patchesXY[0], patchesXY[1]) - # argsTiltXCorr += ' -OverlapOfPatchesXandY %s ' % OverlapOfPatchesXandY - # else: - # numberPatchesXY = self.numberOfPatches.getListFromValues() - # argsTiltXCorr += ' -NumberOfPatchesXandY %d,%d ' % (numberPatchesXY[0], numberPatchesXY[1]) - # - # Plugin.runImod(self, 'tiltxcorr', argsTiltXCorr % paramsTiltXCorr) - # - # def chopcontsStep(self, tsObjId): - # ''' - # $imodchopconts -StandardInput - # InputModel cryo_pt.fid - # OutputModel cryo.fid - # MinimumOverlap 4 - # AssignSurfaces 1 - # ''' - # - # ts = self._getTiltSeries(tsObjId) - # tsId = ts.getTsId() - # extraPrefix = self._getExtraPath(tsId) - # - # firstItem = ts.getFirstItem() - # MinimumOverlap = 4 - # AssignSurfaces = 1 - # LengthOfPieces = -1 - # - # paramschopconts = { - # 'inputFile': os.path.join(extraPrefix, firstItem.parseFileName(suffix="_pt", extension=".fid")), - # 'outputFile': os.path.join(extraPrefix, firstItem.parseFileName(suffix="_gaps", extension=".fid")), - # 'MinimumOverlap': MinimumOverlap, - # 'AssignSurfaces': AssignSurfaces, - # 'LengthOfPieces': LengthOfPieces - # } - # argschopconts = " " \ - # "-InputModel %(inputFile)s " \ - # "-OutputModel %(outputFile)s " \ - # "-MinimumOverlap %(MinimumOverlap)s " \ - # "-AssignSurfaces %(AssignSurfaces)s " \ - # "-LengthOfPieces %(LengthOfPieces)s " - # - # Plugin.runImod(self, 'imodchopconts', argschopconts % paramschopconts) + def xcorrStep(self, tsId): + """ + Imod uses the next command line for the xcorr alignment + $tiltxcorr -StandardInput + InputFile cryo_preali.mrc + OutputFile cryo_pt.fid + RotationAngle -12.6 + TiltFile cryo.rawtlt + FilterRadius2 0.125 + FilterSigma1 0.03 + FilterSigma2 0.03 + BordersInXandY 102,102 + IterateCorrelations 1 + SizeOfPatchesXandY 680,680 + OverlapOfPatchesXandY 0.33,0.33 + PrealignmentTransformFile cryo.prexg + ImagesAreBinned 1 + """ + ts = self.tsDict[tsId] + angleFilePath = self.getExtraOutFile(tsId, ext=TLT_EXT) + xfFile = self.getExtraOutFile(tsId, ext=XF_EXT) + ts.writeXfFile(xfFile) + + borders = self.pxTrim.getListFromValues() + sizePatches = self.sizeOfPatches.getListFromValues() + + BordersInXandY = '%d,%d' % (borders[0], borders[1]) + SizeOfPatchesXandY = '%d,%d' % (sizePatches[0], sizePatches[1]) + + paramsTiltXCorr = { + 'inputFile': self.getTmpOutFile(tsId), + 'outputFile': self.getExtraOutFile(tsId, suffix="pt", ext=".fid"), + 'RotationAngle': ts.getAcquisition().getTiltAxisAngle(), + 'TiltFile': angleFilePath, + 'FilterRadius2': self.filterRadius2.get(), + 'FilterSigma1': self.filterSigma1.get(), + 'FilterSigma2': self.filterSigma2.get(), + 'BordersInXandY': BordersInXandY, + 'IterateCorrelations': self.iterationsSubpixel.get(), + 'SizeOfPatchesXandY': SizeOfPatchesXandY, + 'PrealignmentTransformFile': xfFile, + + 'ImagesAreBinned': 1, + } + argsTiltXCorr = " " \ + "-InputFile %(inputFile)s " \ + "-OutputFile %(outputFile)s " \ + "-RotationAngle %(RotationAngle)s " \ + "-TiltFile %(TiltFile)s " \ + "-FilterRadius2 %(FilterRadius2)s " \ + "-FilterSigma1 %(FilterSigma1)s " \ + "-FilterSigma2 %(FilterSigma2)s " \ + "-BordersInXandY %(BordersInXandY)s " \ + "-IterateCorrelations %(IterateCorrelations)s " \ + "-SizeOfPatchesXandY %(SizeOfPatchesXandY)s " \ + "-PrealignmentTransformFile %(PrealignmentTransformFile)s " \ + "-ImagesAreBinned %(ImagesAreBinned)s " + + if self.patchLayout.get() == 0: + patchesXY = self.overlapPatches.getListFromValues(caster=float) + OverlapOfPatchesXandY = '%f,%f' % (patchesXY[0], patchesXY[1]) + argsTiltXCorr += ' -OverlapOfPatchesXandY %s ' % OverlapOfPatchesXandY + else: + numberPatchesXY = self.numberOfPatches.getListFromValues() + argsTiltXCorr += ' -NumberOfPatchesXandY %d,%d ' % (numberPatchesXY[0], numberPatchesXY[1]) + + Plugin.runImod(self, 'tiltxcorr', argsTiltXCorr % paramsTiltXCorr) + + def chopcontsStep(self, tsId): + """ + $imodchopconts -StandardInput + InputModel cryo_pt.fid + OutputModel cryo.fid + MinimumOverlap 4 + AssignSurfaces 1 + """ + MinimumOverlap = 4 + AssignSurfaces = 1 + LengthOfPieces = -1 + + paramschopconts = { + 'inputFile': self.getExtraOutFile(tsId, suffix="pt", ext=".fid"), + 'outputFile': self.getExtraOutFile(tsId, suffix="gaps", ext=".fid"), + 'MinimumOverlap': MinimumOverlap, + 'AssignSurfaces': AssignSurfaces, + 'LengthOfPieces': LengthOfPieces + } + argschopconts = " " \ + "-InputModel %(inputFile)s " \ + "-OutputModel %(outputFile)s " \ + "-MinimumOverlap %(MinimumOverlap)s " \ + "-AssignSurfaces %(AssignSurfaces)s " \ + "-LengthOfPieces %(LengthOfPieces)s " + + Plugin.runImod(self, 'imodchopconts', argschopconts % paramschopconts) def createOutputStep(self): if self.FiducialModelGaps: From 06c62ee77394853eabe62f923bf7406894286969 Mon Sep 17 00:00:00 2001 From: JorMaister Date: Fri, 19 Apr 2024 11:08:46 +0200 Subject: [PATCH 27/49] update CHANGES.txt --- CHANGES.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.txt b/CHANGES.txt index 039d400e..1a36ef8b 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -10,6 +10,8 @@ * The xf, defocus, and tlt files are generated inside each TS extra sub-directory. Developers: - Update the acquisition order in the CTFTomo objects (field added to that class in scipion-em-tomo v3.7.0). + - Refactorization: avoid the excesive usage of getFirstItem. + - File generation methods (most of them in utils) adapted to the CTF-TS intersection functionality. 3.3.0: - bugfix for import ctf, set missing defocus flag - move plugin-specific import CTF protocol to the core From 4652e29ca8cfd66422a845257779738c736bf189 Mon Sep 17 00:00:00 2001 From: JorMaister Date: Mon, 22 Apr 2024 15:55:28 +0200 Subject: [PATCH 28/49] Some more extension centralization. Tests passing --- imod/protocols/protocol_base.py | 15 ++++++++-- .../protocol_ctfEstimation_automatic.py | 6 ++-- imod/protocols/protocol_fiducialAlignment.py | 28 +++++++++---------- imod/protocols/protocol_fiducialModel.py | 28 +++++++++---------- imod/protocols/protocol_goldBeadPicker3d.py | 10 +++---- imod/protocols/protocol_tomoReconstruction.py | 10 +++---- imod/protocols/protocol_xCorrPrealignment.py | 12 ++++---- imod/protocols/protocol_xRaysEraser.py | 4 +-- imod/tests/test_protocols_imod.py | 22 +++++++-------- 9 files changed, 72 insertions(+), 63 deletions(-) diff --git a/imod/protocols/protocol_base.py b/imod/protocols/protocol_base.py index 86cfd60c..6cc952f3 100644 --- a/imod/protocols/protocol_base.py +++ b/imod/protocols/protocol_base.py @@ -58,6 +58,15 @@ XF_EXT = 'xf' TLT_EXT = 'tlt' DEFOCUS_EXT = 'defocus' +FID_EXT = 'fid' +TXT_EXT = 'txt' +REC_EXT = 'rec' +XYZ_EXT = 'xyz' +MOD_EXT = 'mod' +SEED_EXT = 'seed' +PREXF_EXT = 'prexf' +PREXG_EXT = 'prexg' +SFID_EXT = 'sfid' class ProtImodBase(ProtTomoImportFiles, EMProtocol, ProtTomoBase): @@ -246,13 +255,13 @@ def genTsPaths(self, tsId): path.makePath(*[self._getExtraPath(tsId), self._getTmpPath(tsId)]) @staticmethod - def getOutTsFileName(tsId, suffix=None, ext='st'): + def getOutTsFileName(tsId, suffix=None, ext='mrc'): return f'{tsId}_{suffix}.{ext}' if suffix else f'{tsId}.{ext}' - def getTmpOutFile(self, tsId, suffix=None, ext='st'): + def getTmpOutFile(self, tsId, suffix=None, ext='mrc'): return self._getTmpPath(tsId, self.getOutTsFileName(tsId, suffix=suffix, ext=ext)) - def getExtraOutFile(self, tsId, suffix=None, ext='st'): + def getExtraOutFile(self, tsId, suffix=None, ext='mrc'): return self._getExtraPath(tsId, self.getOutTsFileName(tsId, suffix=suffix, ext=ext)) def convertInputStep(self, tsObjId, generateAngleFile=True, imodInterpolation=True, doSwap=False, diff --git a/imod/protocols/protocol_ctfEstimation_automatic.py b/imod/protocols/protocol_ctfEstimation_automatic.py index bbac5a12..884aca05 100644 --- a/imod/protocols/protocol_ctfEstimation_automatic.py +++ b/imod/protocols/protocol_ctfEstimation_automatic.py @@ -32,7 +32,7 @@ import tomo.objects as tomoObj from .. import Plugin, utils -from .protocol_base import ProtImodBase, OUTPUT_CTF_SERIE, TLT_EXT +from .protocol_base import ProtImodBase, OUTPUT_CTF_SERIE, TLT_EXT, DEFOCUS_EXT class ProtImodAutomaticCtfEstimation(ProtImodBase): @@ -326,7 +326,7 @@ def ctfEstimation(self, tsId, expDefoci): paramsCtfPlotter = { 'inputStack': self.getTmpOutFile(tsId), 'angleFile': self.getExtraOutFile(tsId, ext=TLT_EXT), - 'defocusFile': self.getExtraOutFile(tsId, ext="defocus"), + 'defocusFile': self.getExtraOutFile(tsId, ext=DEFOCUS_EXT), 'axisAngle': ts.getAcquisition().getTiltAxisAngle(), 'pixelSize': self.sRate / 10, # nm 'voltage': self.acq.getVoltage(), @@ -437,7 +437,7 @@ def ctfEstimation(self, tsId, expDefoci): def createOutputStep(self, tsId, outputSetName=OUTPUT_CTF_SERIE): ts = self.tsDict[tsId] - defocusFilePath = self.getExtraOutFile(tsId, ext="defocus") + defocusFilePath = self.getExtraOutFile(tsId, ext=DEFOCUS_EXT) if os.path.exists(defocusFilePath): output = self.getOutputSetOfCTFTomoSeries(outputSetName) defocusFileFlag = utils.getDefocusFileFlag(defocusFilePath) diff --git a/imod/protocols/protocol_fiducialAlignment.py b/imod/protocols/protocol_fiducialAlignment.py index cd9885e0..ee521989 100644 --- a/imod/protocols/protocol_fiducialAlignment.py +++ b/imod/protocols/protocol_fiducialAlignment.py @@ -36,7 +36,7 @@ TiltSeries, TiltSeriesCoordinate) from .. import Plugin, utils -from .protocol_base import ProtImodBase, TLT_EXT, XF_EXT +from .protocol_base import ProtImodBase, TLT_EXT, XF_EXT, FID_EXT, TXT_EXT, XYZ_EXT, MOD_EXT, SFID_EXT class ProtImodFiducialAlignment(ProtImodBase): @@ -361,13 +361,13 @@ def computeFiducialAlignmentStep(self, tsId): 'imageFile': self.getTmpOutFile(tsId), 'imagesAreBinned': 1, 'unbinnedPixelSize': ts.getSamplingRate() / 10, - 'outputModelFile': self.getExtraOutFile(tsId, suffix="fidxyz", ext="mod"), - 'outputResidualFile': self.getExtraOutFile(tsId, suffix="resid", ext="txt"), - 'outputFidXYZFile': self.getExtraOutFile(tsId, suffix="fid", ext="xyz"), + 'outputModelFile': self.getExtraOutFile(tsId, suffix="fidxyz", ext=MOD_EXT), + 'outputResidualFile': self.getExtraOutFile(tsId, suffix="resid", ext=TXT_EXT), + 'outputFidXYZFile': self.getExtraOutFile(tsId, suffix="fid", ext=XYZ_EXT), 'outputTiltFile': self.getExtraOutFile(tsId, suffix="interpolated", ext=TLT_EXT), 'outputXAxisTiltFile': self.getExtraOutFile(tsId, ext="xtilt"), - 'outputTransformFile': self.getExtraOutFile(tsId, suffix="fid", ext="xf"), - 'outputFilledInModel': self.getExtraOutFile(tsId, suffix="noGaps", ext="fid"), + 'outputTransformFile': self.getExtraOutFile(tsId, suffix="fid", ext=XF_EXT), + 'outputFilledInModel': self.getExtraOutFile(tsId, suffix="noGaps", ext=FID_EXT), 'rotationAngle': ts.getAcquisition().getTiltAxisAngle(), 'tiltFile': self.getExtraOutFile(tsId, ext=TLT_EXT), 'angleOffset': 0.0, @@ -483,11 +483,11 @@ def computeFiducialAlignmentStep(self, tsId): @ProtImodBase.tryExceptDecorator def translateFiducialPointModelStep(self, tsId): # Check that previous steps have been completed satisfactorily - noGapsFid = self.getExtraOutFile(tsId, suffix="noGaps", ext="fid") + noGapsFid = self.getExtraOutFile(tsId, suffix="noGaps", ext=FID_EXT) if os.path.exists(noGapsFid): paramsNoGapModel2Point = { 'inputFile': noGapsFid, - 'outputFile': self.getExtraOutFile(tsId, suffix="noGaps_fid", ext="txt") + 'outputFile': self.getExtraOutFile(tsId, suffix="noGaps_fid", ext=TXT_EXT) } argsNoGapModel2Point = "-InputFile %(inputFile)s " \ "-OutputFile %(outputFile)s" @@ -630,7 +630,7 @@ def eraseGoldBeadsStep(self, tsId): paramsCcderaser = { 'inputFile': self.getTmpOutFile(tsId), 'outputFile': self.getExtraOutFile(tsId), - 'modelFile': self.getExtraOutFile(tsId, suffix="noGaps", ext="fid"), + 'modelFile': self.getExtraOutFile(tsId, suffix="noGaps", ext=FID_EXT), 'betterRadius': self.betterRadius.get() / 2, 'polynomialOrder': 0, 'circleObjects': "/" @@ -655,14 +655,14 @@ def computeOutputModelsStep(self, tsId): tsId = ts.getTsId() # Create the output set of landmark models with no gaps - fiducialNoGapFilePath = self.getExtraOutFile(tsId, suffix="noGaps_fid", ext='txt') + fiducialNoGapFilePath = self.getExtraOutFile(tsId, suffix="noGaps_fid", ext=TXT_EXT) if os.path.exists(fiducialNoGapFilePath): output = self.getOutputFiducialModelNoGaps() output.setSetOfTiltSeries(self.inputSetOfTiltSeries.get()) fiducialNoGapList = utils.formatFiducialList(fiducialNoGapFilePath) - fiducialModelNoGapPath = self.getExtraOutFile(tsId, suffix="noGaps", ext="fid") - landmarkModelNoGapsFilePath = self.getExtraOutFile(tsId, suffix="noGaps", ext="sfid") - landmarkModelNoGapsResidPath = self.getExtraOutFile(tsId, suffix="resid", ext="txt") + fiducialModelNoGapPath = self.getExtraOutFile(tsId, suffix="noGaps", ext=FID_EXT) + landmarkModelNoGapsFilePath = self.getExtraOutFile(tsId, suffix="noGaps", ext=SFID_EXT) + landmarkModelNoGapsResidPath = self.getExtraOutFile(tsId, suffix="resid", ext=TXT_EXT) fiducialNoGapsResidList = utils.formatFiducialResidList(landmarkModelNoGapsResidPath) @@ -705,7 +705,7 @@ def computeOutputModelsStep(self, tsId): output.write() # Create the output set of 3D coordinates - coordFilePath = self.getExtraOutFile(tsId, suffix="fid", ext="xyz") + coordFilePath = self.getExtraOutFile(tsId, suffix="fid", ext=XYZ_EXT) if os.path.exists(coordFilePath): diff --git a/imod/protocols/protocol_fiducialModel.py b/imod/protocols/protocol_fiducialModel.py index 0c164718..3e53ad52 100644 --- a/imod/protocols/protocol_fiducialModel.py +++ b/imod/protocols/protocol_fiducialModel.py @@ -33,7 +33,7 @@ import tomo.objects as tomoObj from .. import Plugin, utils -from .protocol_base import ProtImodBase, TLT_EXT, XF_EXT +from .protocol_base import ProtImodBase, TLT_EXT, XF_EXT, FID_EXT, TXT_EXT, SEED_EXT, SFID_EXT class ProtImodFiducialModel(ProtImodBase): @@ -227,8 +227,8 @@ def generateTrackComStep(self, tsId): paramsDict = { 'imageFile': self.getTmpOutFile(tsId), - 'inputSeedModel': self.getExtraOutFile(tsId, ext="seed"), - 'outputModel': self.getExtraOutFile(tsId, suffix="gaps", ext="fid"), + 'inputSeedModel': self.getExtraOutFile(tsId, ext=SEED_EXT), + 'outputModel': self.getExtraOutFile(tsId, suffix="gaps", ext=FID_EXT), 'tiltFile': self.getExtraOutFile(tsId, ext=TLT_EXT), 'rotationAngle': ts.getAcquisition().getTiltAxisAngle(), 'fiducialDiameter': fiducialDiameterPixel, @@ -282,8 +282,8 @@ def generateFiducialModelStep(self, tsId): scaling = fiducialDiameterPixel / 12.5 if fiducialDiameterPixel > 12.5 else 1 paramsBeadtrack = { - 'inputSeedModel': self.getExtraOutFile(tsId, ext="seed"), - 'outputModel': self.getExtraOutFile(tsId, suffix="gaps", ext="fid"), + 'inputSeedModel': self.getExtraOutFile(tsId, ext=SEED_EXT), + 'outputModel': self.getExtraOutFile(tsId, suffix="gaps", ext=FID_EXT), 'imageFile': self.getTmpOutFile(tsId), 'imagesAreBinned': 1, 'tiltFile': self.getExtraOutFile(tsId, ext=TLT_EXT), @@ -363,7 +363,7 @@ def generateFiducialModelStep(self, tsId): if self.doTrackWithModel: # repeat tracking with the current model as seed path.copyFile(paramsBeadtrack['inputSeedModel'], - self.getExtraOutFile(tsId, suffix="orig", ext="seed")) + self.getExtraOutFile(tsId, suffix="orig", ext=SEED_EXT)) path.moveFile(paramsBeadtrack['outputModel'], paramsBeadtrack['inputSeedModel']) @@ -373,11 +373,11 @@ def translateFiducialPointModelStep(self, tsId): ts = self.tsDict[tsId] # Check that previous steps have been completed satisfactorily - gapsFidFile = self.getExtraOutFile(tsId, suffix='gaps', ext='fid') + gapsFidFile = self.getExtraOutFile(tsId, suffix='gaps', ext=FID_EXT) if os.path.exists(gapsFidFile): paramsGapModel2Point = { 'inputFile': gapsFidFile, - 'outputFile': self.getExtraOutFile(tsId, suffix="gaps_fid", ext="txt") + 'outputFile': self.getExtraOutFile(tsId, suffix="gaps_fid", ext=TXT_EXT) } argsGapModel2Point = "-InputFile %(inputFile)s " \ "-OutputFile %(outputFile)s" @@ -389,11 +389,11 @@ def computeOutputModelsStep(self, tsId): # Create the output set of landmark models with gaps # Check that previous steps have been completed satisfactorily - fiducialModelGapPath = self.getExtraOutFile(tsId, suffix='gaps', ext='fid') + fiducialModelGapPath = self.getExtraOutFile(tsId, suffix='gaps', ext=FID_EXT) if os.path.exists(fiducialModelGapPath): output = self.getOutputFiducialModelGaps() - landmarkModelGapsFilePath = self.getExtraOutFile(tsId, suffix='gaps', ext='sfid') - fiducialModelGapTxtPath = self.getExtraOutFile(tsId, suffix="gaps_fid", ext="txt") + landmarkModelGapsFilePath = self.getExtraOutFile(tsId, suffix='gaps', ext=SFID_EXT) + fiducialModelGapTxtPath = self.getExtraOutFile(tsId, suffix="gaps_fid", ext=TXT_EXT) fiducialGapList = utils.formatFiducialList(fiducialModelGapTxtPath) fiducialDiameterPixel = self.fiducialDiameter.get() / ( @@ -459,7 +459,7 @@ def xcorrStep(self, tsId): paramsTiltXCorr = { 'inputFile': self.getTmpOutFile(tsId), - 'outputFile': self.getExtraOutFile(tsId, suffix="pt", ext=".fid"), + 'outputFile': self.getExtraOutFile(tsId, suffix="pt", ext=FID_EXT), 'RotationAngle': ts.getAcquisition().getTiltAxisAngle(), 'TiltFile': angleFilePath, 'FilterRadius2': self.filterRadius2.get(), @@ -509,8 +509,8 @@ def chopcontsStep(self, tsId): LengthOfPieces = -1 paramschopconts = { - 'inputFile': self.getExtraOutFile(tsId, suffix="pt", ext=".fid"), - 'outputFile': self.getExtraOutFile(tsId, suffix="gaps", ext=".fid"), + 'inputFile': self.getExtraOutFile(tsId, suffix="pt", ext=FID_EXT), + 'outputFile': self.getExtraOutFile(tsId, suffix="gaps", ext=FID_EXT), 'MinimumOverlap': MinimumOverlap, 'AssignSurfaces': AssignSurfaces, 'LengthOfPieces': LengthOfPieces diff --git a/imod/protocols/protocol_goldBeadPicker3d.py b/imod/protocols/protocol_goldBeadPicker3d.py index a8de4636..0c727160 100644 --- a/imod/protocols/protocol_goldBeadPicker3d.py +++ b/imod/protocols/protocol_goldBeadPicker3d.py @@ -34,7 +34,7 @@ import tomo.constants as constants from .. import Plugin, utils -from .protocol_base import ProtImodBase +from .protocol_base import ProtImodBase, XYZ_EXT, MOD_EXT class ProtImodGoldBeadPicker3d(ProtImodBase): @@ -139,7 +139,7 @@ def pickGoldBeadsStep(self, tsId): """ Run findbeads3d IMOD program """ paramsFindbeads3d = { 'inputFile': tomo.getFileName(), - 'outputFile': self.getExtraOutFile(tsId, ext='mod'), + 'outputFile': self.getExtraOutFile(tsId, ext=MOD_EXT), 'beadSize': self.beadDiameter.get(), 'minRelativeStrength': self.minRelativeStrength.get(), 'minSpacing': self.minSpacing.get(), @@ -160,8 +160,8 @@ def pickGoldBeadsStep(self, tsId): def convertModelToCoordinatesStep(self, tsId): """ Run model2point IMOD program """ paramsModel2Point = { - 'inputFile': self.getExtraOutFile(tsId, ext='mod'), - 'outputFile': self.getExtraOutFile(tsId, ext='xyz'), + 'inputFile': self.getExtraOutFile(tsId, ext=MOD_EXT), + 'outputFile': self.getExtraOutFile(tsId, ext=XYZ_EXT), } argsModel2Point = "-InputFile %(inputFile)s " \ @@ -176,7 +176,7 @@ def createOutputStep(self, tsId): output = self.getOutputSetOfCoordinates3Ds(self.inputSetOfTomograms.get(), self.inputSetOfTomograms.get()) - coordFilePath = self.getExtraOutFile(tsId, ext='xyz') + coordFilePath = self.getExtraOutFile(tsId, ext=XYZ_EXT) coordList = utils.formatGoldBead3DCoordinatesList(coordFilePath) with self._lock: diff --git a/imod/protocols/protocol_tomoReconstruction.py b/imod/protocols/protocol_tomoReconstruction.py index 0f225d95..97617af6 100644 --- a/imod/protocols/protocol_tomoReconstruction.py +++ b/imod/protocols/protocol_tomoReconstruction.py @@ -33,7 +33,7 @@ from .. import Plugin from .protocol_base import (ProtImodBase, EXT_MRC_ODD_NAME, EXT_MRC_EVEN_NAME, - EXT_MRCS_TS_EVEN_NAME, EXT_MRCS_TS_ODD_NAME, TLT_EXT, ODD, MRCS_EXT, EVEN, MRC_EXT) + EXT_MRCS_TS_EVEN_NAME, EXT_MRCS_TS_ODD_NAME, TLT_EXT, ODD, MRCS_EXT, EVEN, MRC_EXT, REC_EXT) class ProtImodTomoReconstruction(ProtImodBase): @@ -261,7 +261,7 @@ def computeReconstructionStep(self, tsId): ts = self.tsDict[tsId] paramsTilt = { 'InputProjections': self.getTmpOutFile(tsId), - 'OutputFile': self.getTmpOutFile(tsId, ext="rec"), + 'OutputFile': self.getTmpOutFile(tsId, ext=REC_EXT), 'TiltFile': self.getExtraOutFile(tsId, ext=TLT_EXT), 'Thickness': self.tomoThickness.get(), 'FalloffIsTrueSigma': 1, @@ -313,17 +313,17 @@ def getArgs(): if self.applyToOddEven(ts): paramsTilt['InputProjections'] = self.getExtraOutFile(tsId, suffix=ODD, ext=MRCS_EXT) - oddEvenTmp[0] = self.getExtraOutFile(tsId, suffix=ODD, ext="rec") + oddEvenTmp[0] = self.getExtraOutFile(tsId, suffix=ODD, ext=REC_EXT) paramsTilt['OutputFile'] = oddEvenTmp[0] Plugin.runImod(self, 'tilt', argsTilt % paramsTilt) paramsTilt['InputProjections'] = self.getExtraOutFile(tsId, suffix=EVEN, ext=MRCS_EXT) - oddEvenTmp[1] = self.getExtraOutFile(tsId, suffix=EVEN, ext="rec") + oddEvenTmp[1] = self.getExtraOutFile(tsId, suffix=EVEN, ext=REC_EXT) paramsTilt['OutputFile'] = oddEvenTmp[1] Plugin.runImod(self, 'tilt', argsTilt % paramsTilt) paramsTrimVol = { - 'input': self.getTmpOutFile(tsId, ext="rec"), + 'input': self.getTmpOutFile(tsId, ext=REC_EXT), 'output': self.getExtraOutFile(tsId, ext=MRC_EXT), 'options': getArgs() } diff --git a/imod/protocols/protocol_xCorrPrealignment.py b/imod/protocols/protocol_xCorrPrealignment.py index 94b07d47..6c4e07d1 100644 --- a/imod/protocols/protocol_xCorrPrealignment.py +++ b/imod/protocols/protocol_xCorrPrealignment.py @@ -35,7 +35,7 @@ import tomo.objects as tomoObj from .. import Plugin, utils -from .protocol_base import ProtImodBase, TLT_EXT +from .protocol_base import ProtImodBase, TLT_EXT, PREXF_EXT, PREXG_EXT class ProtImodXcorrPrealignment(ProtImodBase): @@ -150,7 +150,7 @@ def computeXcorrStep(self, tsId): paramsXcorr = { 'input': self.getTmpOutFile(tsId), - 'output': self.getExtraOutFile(tsId, ext='prexf'), + 'output': self.getExtraOutFile(tsId, ext=PREXF_EXT), 'tiltfile': self.getExtraOutFile(tsId, ext=TLT_EXT), 'rotationAngle': tiltAxisAngle, 'filterSigma1': self.filterSigma1.get(), @@ -198,8 +198,8 @@ def computeXcorrStep(self, tsId): Plugin.runImod(self, 'tiltxcorr', argsXcorr % paramsXcorr) paramsXftoxg = { - 'input': self.getExtraOutFile(tsId, ext='prexf'), - 'goutput': self.getExtraOutFile(tsId, ext='prexg'), + 'input': self.getExtraOutFile(tsId, ext=PREXF_EXT), + 'goutput': self.getExtraOutFile(tsId, ext=PREXG_EXT), } argsXftoxg = "-input %(input)s " \ "-NumberToFit 0 " \ @@ -210,7 +210,7 @@ def generateOutputStackStep(self, tsId): """ Generate tilt-serie with the associated transform matrix """ ts = self.tsDict[tsId] output = self.getOutputSetOfTiltSeries(self.inputSetOfTiltSeries.get()) - alignmentMatrix = utils.formatTransformationMatrix(self.getExtraOutFile(tsId, ext='prexg')) + alignmentMatrix = utils.formatTransformationMatrix(self.getExtraOutFile(tsId, ext=PREXG_EXT)) newTs = tomoObj.TiltSeries(tsId=tsId) newTs.copyInfo(ts) newTs.getAcquisition().setTiltAxisAngle(self.getTiltAxisOrientation(ts)) @@ -256,7 +256,7 @@ def computeInterpolatedStackStep(self, tsId): paramsAlignment = { 'input': self.getTmpOutFile(tsId), 'output': self.getExtraOutFile(tsId), - 'xform': self.getExtraOutFile(tsId, ext='prexg'), + 'xform': self.getExtraOutFile(tsId, ext=PREXG_EXT), 'bin': self.binning.get(), 'imagebinned': 1.0 } diff --git a/imod/protocols/protocol_xRaysEraser.py b/imod/protocols/protocol_xRaysEraser.py index d28792bc..d21fb24f 100644 --- a/imod/protocols/protocol_xRaysEraser.py +++ b/imod/protocols/protocol_xRaysEraser.py @@ -34,7 +34,7 @@ from .. import Plugin from .protocol_base import ProtImodBase, OUTPUT_TILTSERIES_NAME, EXT_MRCS_TS_ODD_NAME, EXT_MRCS_TS_EVEN_NAME, ODD, \ - MRCS_EXT, EVEN + MRCS_EXT, EVEN, MOD_EXT class ProtImodXraysEraser(ProtImodBase): @@ -216,7 +216,7 @@ def eraseXraysStep(self, tsId): 'annulusWidth': 2.0, 'xyScanSize': 100, 'edgeExclusionWidth': 4, - 'pointModel': self.getExtraOutFile(tsId, suffix="fid", ext="mod"), + 'pointModel': self.getExtraOutFile(tsId, suffix="fid", ext=MOD_EXT), 'borderSize': 2, 'polynomialOrder': 2, } diff --git a/imod/tests/test_protocols_imod.py b/imod/tests/test_protocols_imod.py index d102ffc8..46ef956d 100644 --- a/imod/tests/test_protocols_imod.py +++ b/imod/tests/test_protocols_imod.py @@ -460,7 +460,7 @@ def test_doseFilterOutputTS(self): tsId = ts.getFirstItem().getTsId() self.assertTrue(os.path.exists(os.path.join(self.protDoseFilter._getExtraPath(tsId), - tsId + ".st"))) + tsId + ".mrc"))) def test_xRaysEraserOutputTS(self): @@ -470,7 +470,7 @@ def test_xRaysEraserOutputTS(self): tsId = ts.getFirstItem().getTsId() self.assertTrue(os.path.exists(os.path.join(self.protXRaysEraser._getExtraPath(tsId), - tsId + ".st"))) + tsId + ".mrc"))) def test_excludeViewsOutputTS(self): @@ -480,7 +480,7 @@ def test_excludeViewsOutputTS(self): tsId = ts.getFirstItem().getTsId() self.assertTrue(os.path.exists(os.path.join(self.protExcludeViews._getExtraPath(tsId), - tsId + ".st"))) + tsId + ".mrc"))) for index, tsOut in enumerate(ts): self.assertEqual(tsOut.getSize(), self.excludeViewsOutputSizes[tsOut.getTsId()]) @@ -493,7 +493,7 @@ def test_normalizationOutputTS(self): tsId = ts.getFirstItem().getTsId() self.assertTrue(os.path.exists(os.path.join(self.protTSNormalization._getExtraPath(tsId), - tsId + ".st"))) + tsId + ".mrc"))) inSamplingRate = self.protTSNormalization.inputSetOfTiltSeries.get().getSamplingRate() outSamplingRate = ts.getSamplingRate() @@ -507,7 +507,7 @@ def test_prealignmentOutputTS(self): tsId = ts.getFirstItem().getTsId() outputLocation = os.path.join(self.protXcorr._getExtraPath(tsId), - tsId + ".st") + tsId + ".mrc") self.assertTrue(os.path.exists(outputLocation)) @@ -521,7 +521,7 @@ def test_prealignmentOutputInterpolatedTS(self): tsId = ts.getFirstItem().getTsId() outputLocation = os.path.join(self.protXcorr._getExtraPath(tsId), - tsId + ".st") + tsId + ".mrc") self.assertTrue(os.path.exists(outputLocation)) @@ -552,7 +552,7 @@ def test_fiducialAlignmentOutputTS(self): tsId = output.getFirstItem().getTsId() outputLocation = os.path.join(self.protFiducialAlignment._getExtraPath(tsId), - tsId + ".st") + tsId + ".mrc") self.assertTrue(os.path.exists(outputLocation)) @@ -565,7 +565,7 @@ def test_fiducialAlignmentOutputTS(self): tsId = output.getFirstItem().getTsId() outputLocation = os.path.join(self.protFiducialAlignment._getExtraPath(tsId), - tsId + ".st") + tsId + ".mrc") self.assertTrue(os.path.exists(outputLocation)) @@ -608,7 +608,7 @@ def test_applyTransformationMatrixOutputInterpolatedTS(self): tsId = output.getFirstItem().getTsId() outputLocation = os.path.join(self.protApplyTransformationMatrix._getExtraPath(tsId), - tsId + ".st") + tsId + ".mrc") self.assertTrue(os.path.exists(outputLocation)) @@ -692,7 +692,7 @@ def setUpClass(cls): # Create links to the input tilt-series and its associated mdoc file to test the protocols with a set of two # elements to make the tests more robust linkTs = os.path.join(os.path.split(cls.inputSoTS)[0], - "WTI042413_1series4_copy.st") + "WTI042413_1series4_copy.mrc") if not os.path.exists(linkTs): path.createLink(cls.inputSoTS, linkTs) @@ -765,7 +765,7 @@ def test_ctfCorrectionOutput(self): for ts in output: tsId = ts.getTsId() outputLocation = os.path.join(self.protCTFCorrection._getExtraPath(tsId), - '%s.st' % tsId) + '%s.mrc' % tsId) self.assertTrue(os.path.exists(outputLocation)) From 4cce2f52731a85dbb0f20496b10de6de98d00111 Mon Sep 17 00:00:00 2001 From: Ricardo Righetto Date: Tue, 14 May 2024 20:51:03 +0200 Subject: [PATCH 29/49] Added tapering by default and option for linear interpolation --- .../protocol_applyTransformationMatrix.py | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/imod/protocols/protocol_applyTransformationMatrix.py b/imod/protocols/protocol_applyTransformationMatrix.py index 18a8f998..d09f0301 100644 --- a/imod/protocols/protocol_applyTransformationMatrix.py +++ b/imod/protocols/protocol_applyTransformationMatrix.py @@ -63,14 +63,23 @@ def _defineParams(self, form): form.addParam('binning', params.IntParam, default=1, label='Binning for the interpolated', - help='Binning to be applied to the interpolated tilt-series in IMOD ' + help='Binning to be applied to the interpolated tilt-series in IMOD ' 'convention. \n' 'Binning is an scaling factor given by an integer greater than 1. ' - 'IMOD uses ordinary binning to reduce images in size by the given factor. ' + 'IMOD uses ordinary binning (with antialiasing filter) to reduce images in size by the given factor. ' 'The value of a binned pixel is the average of pixel values in each block ' 'of pixels being binned. Binning is applied before all other image ' 'transformations.') + form.addParam('linear', + params.BooleanParam, + expertLevel=params.LEVEL_ADVANCED, + default=True, + label='Linear interpolation?', + help='From newstack man page: Use linear instead of cubic interpolation to transform images. ' + 'Linear interpolation is more suitable when images are very noisy, ' + 'but cubic interpolation will preserve fine detail better when noise is not an issue.') + form.addParam('processOddEven', params.BooleanParam, expertLevel=params.LEVEL_ADVANCED, @@ -110,7 +119,8 @@ def computeAlignmentStep(self, tsId): 'output': self.getExtraOutFile(tsId), 'xform': self.getExtraOutFile(tsId, ext=XF_EXT), 'bin': binning, - 'imagebinned': 1.0 + 'imagebinned': 1.0, + 'taper': "1,0" } argsAlignment = "-input %(input)s " \ @@ -118,7 +128,11 @@ def computeAlignmentStep(self, tsId): "-xform %(xform)s " \ "-bin %(bin)d " \ "-antialias -1 " \ - "-imagebinned %(imagebinned)s " + "-imagebinned %(imagebinned)s " \ + "-taper %(taper)s " + + if self.linear.get(): + argsAlignment += "-linear " rotationAngle = ts.getAcquisition().getTiltAxisAngle() From d611d9723e9f9aa75010ad736a456de36eb85471 Mon Sep 17 00:00:00 2001 From: Ricardo Righetto Date: Wed, 15 May 2024 10:07:35 +0200 Subject: [PATCH 30/49] Added option for tapering inside or outside and default for linear interpolation is now False --- .../protocol_applyTransformationMatrix.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/imod/protocols/protocol_applyTransformationMatrix.py b/imod/protocols/protocol_applyTransformationMatrix.py index d09f0301..79565edf 100644 --- a/imod/protocols/protocol_applyTransformationMatrix.py +++ b/imod/protocols/protocol_applyTransformationMatrix.py @@ -71,10 +71,18 @@ def _defineParams(self, form): 'of pixels being binned. Binning is applied before all other image ' 'transformations.') - form.addParam('linear', + form.addParam('taperInside', params.BooleanParam, expertLevel=params.LEVEL_ADVANCED, default=True, + label='Taper inwards from the edge?', + help='When the image is transformed areas with no information are filled in (e.g. because of rotation).' + 'Decide whether tapering is done inwards or outwards from the edge.') + + form.addParam('linear', + params.BooleanParam, + expertLevel=params.LEVEL_ADVANCED, + default=False, label='Linear interpolation?', help='From newstack man page: Use linear instead of cubic interpolation to transform images. ' 'Linear interpolation is more suitable when images are very noisy, ' @@ -113,6 +121,7 @@ def computeAlignmentStep(self, tsId): ts = self.tsDict[tsId] firstItem = ts.getFirstItem() binning = self.binning.get() + taperIn = self.taperInside.get() paramsAlignment = { 'input': firstItem.getFileName(), @@ -120,7 +129,7 @@ def computeAlignmentStep(self, tsId): 'xform': self.getExtraOutFile(tsId, ext=XF_EXT), 'bin': binning, 'imagebinned': 1.0, - 'taper': "1,0" + 'taper': "1," + str(int(taperIn)) } argsAlignment = "-input %(input)s " \ @@ -129,7 +138,7 @@ def computeAlignmentStep(self, tsId): "-bin %(bin)d " \ "-antialias -1 " \ "-imagebinned %(imagebinned)s " \ - "-taper %(taper)s " + "-taper %(taper)s " if self.linear.get(): argsAlignment += "-linear " From ff1a67f3c40bec1f55db7d81ddc1d1bb7fd7f121 Mon Sep 17 00:00:00 2001 From: Grigory Sharov Date: Wed, 15 May 2024 09:32:58 +0100 Subject: [PATCH 31/49] simplify a bit --- imod/protocols/protocol_applyTransformationMatrix.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/imod/protocols/protocol_applyTransformationMatrix.py b/imod/protocols/protocol_applyTransformationMatrix.py index 79565edf..a079f2bd 100644 --- a/imod/protocols/protocol_applyTransformationMatrix.py +++ b/imod/protocols/protocol_applyTransformationMatrix.py @@ -121,7 +121,6 @@ def computeAlignmentStep(self, tsId): ts = self.tsDict[tsId] firstItem = ts.getFirstItem() binning = self.binning.get() - taperIn = self.taperInside.get() paramsAlignment = { 'input': firstItem.getFileName(), @@ -129,7 +128,7 @@ def computeAlignmentStep(self, tsId): 'xform': self.getExtraOutFile(tsId, ext=XF_EXT), 'bin': binning, 'imagebinned': 1.0, - 'taper': "1," + str(int(taperIn)) + 'taper': "1,1" if self.taperInside else "1,0" } argsAlignment = "-input %(input)s " \ From 8dbd7bd4ac70831a7b9c0304c269ab4ec71faaa6 Mon Sep 17 00:00:00 2001 From: fede-pe Date: Tue, 21 May 2024 10:18:38 +0200 Subject: [PATCH 32/49] fixing dictionary for CTF correction --- imod/utils.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/imod/utils.py b/imod/utils.py index c0fb80ae..4aed866f 100644 --- a/imod/utils.py +++ b/imod/utils.py @@ -647,8 +647,8 @@ def generateDefocusIMODFileFromObject(ctfTomoSeries, defocusFilePath, lines = [] pattern = "%d\t%d\t%.2f\t%.2f\t%d\n" - for index in defocusUDict.keys(): - + for index in sorted(defocusUDict.keys()): + print(index) if index + nEstimationsInRange > len(defocusUDict.keys()): break @@ -686,7 +686,7 @@ def generateDefocusIMODFileFromObject(ctfTomoSeries, defocusFilePath, # order to match the IMOD defocus file format. lines = ["1\t0\t0.0\t0.0\t0.0\t3\n"] - for index in defocusUDict.keys(): + for index in sorted(defocusUDict.keys()): if index + ctfTomoSeries.getNumberOfEstimationsInRange() > len(defocusUDict.keys()): break @@ -722,7 +722,7 @@ def generateDefocusIMODFileFromObject(ctfTomoSeries, defocusFilePath, # order to match the IMOD defocus file format. lines = ["4\t0\t0.0\t0.0\t0.0\t3\n"] - for index in defocusUDict.keys(): + for index in sorted(defocusUDict.keys()): if index + ctfTomoSeries.getNumberOfEstimationsInRange() > len(defocusUDict.keys()): break @@ -759,7 +759,7 @@ def generateDefocusIMODFileFromObject(ctfTomoSeries, defocusFilePath, # to match the IMOD defocus file format lines = ["5\t0\t0.0\t0.0\t0.0\t3\n"] - for index in defocusUDict.keys(): + for index in sorted(defocusUDict.keys()): if index + ctfTomoSeries.getNumberOfEstimationsInRange() > len(defocusUDict.keys()): break @@ -799,7 +799,7 @@ def generateDefocusIMODFileFromObject(ctfTomoSeries, defocusFilePath, # to match the IMOD defocus file format lines = ["37\t0\t0.0\t0.0\t0.0\t3\n"] - for index in defocusUDict.keys(): + for index in sorted(defocusUDict.keys()): if index + ctfTomoSeries.getNumberOfEstimationsInRange() > len(defocusUDict.keys()): break From bbcf9e2faa65e7039f61d17f305b80361a702fdf Mon Sep 17 00:00:00 2001 From: Vilax Date: Tue, 21 May 2024 10:21:59 +0200 Subject: [PATCH 33/49] Update CHANGES.txt --- CHANGES.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index 1a36ef8b..aecaa7de 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -7,11 +7,12 @@ - The order of the parameters in the forms was slightly modified according to the user actions. - Protocol CTF correction: * Now it can deal with excluded views in the CTF and/or in the TS. - * The xf, defocus, and tlt files are generated inside each TS extra sub-directory. + * The xf, defocus, and tlt files are generated inside each TS extra sub-directory Developers: - Update the acquisition order in the CTFTomo objects (field added to that class in scipion-em-tomo v3.7.0). - Refactorization: avoid the excesive usage of getFirstItem. - File generation methods (most of them in utils) adapted to the CTF-TS intersection functionality. + - Fixing sorted dictionary in ctf correction. How to access the defocus list in utils.py 3.3.0: - bugfix for import ctf, set missing defocus flag - move plugin-specific import CTF protocol to the core From 117af2630e27eb33ac33746f04c79be79864ec06 Mon Sep 17 00:00:00 2001 From: JorMaister Date: Tue, 21 May 2024 12:15:44 +0200 Subject: [PATCH 34/49] Remove print --- CHANGES.txt | 2 +- imod/utils.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index aecaa7de..bd1bff9b 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -7,7 +7,7 @@ - The order of the parameters in the forms was slightly modified according to the user actions. - Protocol CTF correction: * Now it can deal with excluded views in the CTF and/or in the TS. - * The xf, defocus, and tlt files are generated inside each TS extra sub-directory + * The xf, defocus, and tlt files are generated inside each TS extra sub-directory. Developers: - Update the acquisition order in the CTFTomo objects (field added to that class in scipion-em-tomo v3.7.0). - Refactorization: avoid the excesive usage of getFirstItem. diff --git a/imod/utils.py b/imod/utils.py index 4aed866f..e9266008 100644 --- a/imod/utils.py +++ b/imod/utils.py @@ -648,7 +648,6 @@ def generateDefocusIMODFileFromObject(ctfTomoSeries, defocusFilePath, pattern = "%d\t%d\t%.2f\t%.2f\t%d\n" for index in sorted(defocusUDict.keys()): - print(index) if index + nEstimationsInRange > len(defocusUDict.keys()): break From ae5754a6cbf71e466cfcbc18bb569e19ed6b558e Mon Sep 17 00:00:00 2001 From: Ricardo Righetto Date: Mon, 27 May 2024 14:29:59 +0200 Subject: [PATCH 35/49] Protocol was looking for odd/even input stacks in extra, but they are in tmp --- imod/protocols/protocol_tomoReconstruction.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/imod/protocols/protocol_tomoReconstruction.py b/imod/protocols/protocol_tomoReconstruction.py index 97617af6..2aaddec3 100644 --- a/imod/protocols/protocol_tomoReconstruction.py +++ b/imod/protocols/protocol_tomoReconstruction.py @@ -312,12 +312,12 @@ def getArgs(): oddEvenTmp = [[], []] if self.applyToOddEven(ts): - paramsTilt['InputProjections'] = self.getExtraOutFile(tsId, suffix=ODD, ext=MRCS_EXT) + paramsTilt['InputProjections'] = self.getTmpOutFile(tsId, suffix=ODD, ext=MRCS_EXT) oddEvenTmp[0] = self.getExtraOutFile(tsId, suffix=ODD, ext=REC_EXT) paramsTilt['OutputFile'] = oddEvenTmp[0] Plugin.runImod(self, 'tilt', argsTilt % paramsTilt) - paramsTilt['InputProjections'] = self.getExtraOutFile(tsId, suffix=EVEN, ext=MRCS_EXT) + paramsTilt['InputProjections'] = self.getTmpOutFile(tsId, suffix=EVEN, ext=MRCS_EXT) oddEvenTmp[1] = self.getExtraOutFile(tsId, suffix=EVEN, ext=REC_EXT) paramsTilt['OutputFile'] = oddEvenTmp[1] Plugin.runImod(self, 'tilt', argsTilt % paramsTilt) From 8681c730a12538ff20283ae3cbc8a303ab3d854b Mon Sep 17 00:00:00 2001 From: Ricardo Righetto Date: Tue, 28 May 2024 11:10:27 +0200 Subject: [PATCH 36/49] Fixes #268 by removing a mischievous comma --- imod/protocols/protocol_doseFilter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imod/protocols/protocol_doseFilter.py b/imod/protocols/protocol_doseFilter.py index 1278c12f..79545eea 100644 --- a/imod/protocols/protocol_doseFilter.py +++ b/imod/protocols/protocol_doseFilter.py @@ -142,7 +142,7 @@ def doseFilterStep(self, tsId): argsMtffilter += f"-InitialDose {self.initialDose.get():f} " if self.inputDoseType.get() == SCIPION_IMPORT: - outputDoseFilePath = self.getExtraOutFile(tsId, ext="dose"), + outputDoseFilePath = self.getExtraOutFile(tsId, ext="dose") utils.generateDoseFileFromDoseTS(ts, outputDoseFilePath) paramsMtffilter.update({ From ac16ab9d54fae7e1e56b78ad4887b62c83496051 Mon Sep 17 00:00:00 2001 From: Ricardo Righetto Date: Wed, 29 May 2024 18:28:41 +0200 Subject: [PATCH 37/49] Fixed extensions in protocol_tomoReconstruction.py, only uses MRC_EXT now --- imod/protocols/protocol_tomoReconstruction.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/imod/protocols/protocol_tomoReconstruction.py b/imod/protocols/protocol_tomoReconstruction.py index 2aaddec3..583078c7 100644 --- a/imod/protocols/protocol_tomoReconstruction.py +++ b/imod/protocols/protocol_tomoReconstruction.py @@ -261,7 +261,7 @@ def computeReconstructionStep(self, tsId): ts = self.tsDict[tsId] paramsTilt = { 'InputProjections': self.getTmpOutFile(tsId), - 'OutputFile': self.getTmpOutFile(tsId, ext=REC_EXT), + 'OutputFile': self.getTmpOutFile(tsId, ext=MRC_EXT), 'TiltFile': self.getExtraOutFile(tsId, ext=TLT_EXT), 'Thickness': self.tomoThickness.get(), 'FalloffIsTrueSigma': 1, @@ -313,17 +313,17 @@ def getArgs(): if self.applyToOddEven(ts): paramsTilt['InputProjections'] = self.getTmpOutFile(tsId, suffix=ODD, ext=MRCS_EXT) - oddEvenTmp[0] = self.getExtraOutFile(tsId, suffix=ODD, ext=REC_EXT) + oddEvenTmp[0] = self.getExtraOutFile(tsId, suffix=ODD, ext=MRC_EXT) paramsTilt['OutputFile'] = oddEvenTmp[0] Plugin.runImod(self, 'tilt', argsTilt % paramsTilt) paramsTilt['InputProjections'] = self.getTmpOutFile(tsId, suffix=EVEN, ext=MRCS_EXT) - oddEvenTmp[1] = self.getExtraOutFile(tsId, suffix=EVEN, ext=REC_EXT) + oddEvenTmp[1] = self.getExtraOutFile(tsId, suffix=EVEN, ext=MRC_EXT) paramsTilt['OutputFile'] = oddEvenTmp[1] Plugin.runImod(self, 'tilt', argsTilt % paramsTilt) paramsTrimVol = { - 'input': self.getTmpOutFile(tsId, ext=REC_EXT), + 'input': self.getTmpOutFile(tsId, ext=MRC_EXT), 'output': self.getExtraOutFile(tsId, ext=MRC_EXT), 'options': getArgs() } @@ -336,11 +336,11 @@ def getArgs(): if self.applyToOddEven(ts): paramsTrimVol['input'] = oddEvenTmp[0] - paramsTrimVol['output'] = self.getExtraOutFile(tsId, suffix=ODD, ext=MRCS_EXT) + paramsTrimVol['output'] = self.getExtraOutFile(tsId, suffix=ODD, ext=MRC_EXT) Plugin.runImod(self, 'trimvol', argsTrimvol % paramsTrimVol) paramsTrimVol['input'] = oddEvenTmp[1] - paramsTrimVol['output'] = self.getExtraOutFile(tsId, suffix=EVEN, ext=MRCS_EXT) + paramsTrimVol['output'] = self.getExtraOutFile(tsId, suffix=EVEN, ext=MRC_EXT) Plugin.runImod(self, 'trimvol', argsTrimvol % paramsTrimVol) def createOutputStep(self, tsId): @@ -354,8 +354,8 @@ def createOutputStep(self, tsId): newTomogram.setLocation(tomoLocation) if self.applyToOddEven(ts): - halfMapsList = [self.getExtraOutFile(tsId, suffix=ODD, ext=MRCS_EXT), - self.getExtraOutFile(tsId, suffix=EVEN, ext=MRCS_EXT)] + halfMapsList = [self.getExtraOutFile(tsId, suffix=ODD, ext=MRC_EXT), + self.getExtraOutFile(tsId, suffix=EVEN, ext=MRC_EXT)] newTomogram.setHalfMaps(halfMapsList) newTomogram.setTsId(tsId) From fc77902b644e55535f832908dae8480178fb722f Mon Sep 17 00:00:00 2001 From: Grigory Sharov Date: Thu, 30 May 2024 12:45:30 +0100 Subject: [PATCH 38/49] use mrcs for stacks, mrc for volumes --- .../protocol_applyTransformationMatrix.py | 8 ++++---- imod/protocols/protocol_base.py | 16 +++++---------- imod/protocols/protocol_ctfCorrection.py | 12 +++++------ imod/protocols/protocol_doseFilter.py | 8 ++++---- imod/protocols/protocol_tomoPreprocess.py | 6 +++--- imod/protocols/protocol_tomoProjection.py | 9 +++------ imod/protocols/protocol_tomoReconstruction.py | 7 +++---- imod/protocols/protocol_tsPreprocess.py | 8 ++++---- imod/protocols/protocol_xRaysEraser.py | 8 ++++---- imod/tests/test_protocols_imod.py | 20 +++++++++---------- 10 files changed, 46 insertions(+), 56 deletions(-) diff --git a/imod/protocols/protocol_applyTransformationMatrix.py b/imod/protocols/protocol_applyTransformationMatrix.py index a079f2bd..da2f5b98 100644 --- a/imod/protocols/protocol_applyTransformationMatrix.py +++ b/imod/protocols/protocol_applyTransformationMatrix.py @@ -165,10 +165,10 @@ def computeAlignmentStep(self, tsId): oddFn = firstItem.getOdd().split('@')[1] evenFn = firstItem.getEven().split('@')[1] paramsAlignment['input'] = oddFn - paramsAlignment['output'] = self.getExtraOutFile(tsId, suffix=ODD, ext=MRCS_EXT) + paramsAlignment['output'] = self.getExtraOutFile(tsId, suffix=ODD) Plugin.runImod(self, 'newstack', argsAlignment % paramsAlignment) paramsAlignment['input'] = evenFn - paramsAlignment['output'] = self.getExtraOutFile(tsId, suffix=EVEN, ext=MRCS_EXT) + paramsAlignment['output'] = self.getExtraOutFile(tsId, suffix=EVEN) Plugin.runImod(self, 'newstack', argsAlignment % paramsAlignment) def generateOutputStackStep(self, tsId): @@ -203,8 +203,8 @@ def generateOutputStackStep(self, tsId): newTi.setAcquisition(acq) newTi.setLocation(index, outputLocation) if self.applyToOddEven(ts): - locationOdd = index, (self.getExtraOutFile(tsId, suffix=ODD, ext=MRCS_EXT)) - locationEven = index, (self.getExtraOutFile(tsId, suffix=EVEN, ext=MRCS_EXT)) + locationOdd = index, (self.getExtraOutFile(tsId, suffix=ODD)) + locationEven = index, (self.getExtraOutFile(tsId, suffix=EVEN)) newTi.setOddEven([ih.locationToXmipp(locationOdd), ih.locationToXmipp(locationEven)]) else: newTi.setOddEven([]) diff --git a/imod/protocols/protocol_base.py b/imod/protocols/protocol_base.py index 6cc952f3..9f130ded 100644 --- a/imod/protocols/protocol_base.py +++ b/imod/protocols/protocol_base.py @@ -244,24 +244,18 @@ def wrapper(self, tsId, *args): return wrapper - def getTmpTSFile(self, tsId, tmpPrefix=None, suffix=".mrcs"): - if tmpPrefix is None: - tmpPrefix = self._getTmpPath(tsId) - - return os.path.join(tmpPrefix, tsId + suffix) - def genTsPaths(self, tsId): """Generate the subdirectories corresponding to the current tilt-series in tmp and extra""" path.makePath(*[self._getExtraPath(tsId), self._getTmpPath(tsId)]) @staticmethod - def getOutTsFileName(tsId, suffix=None, ext='mrc'): + def getOutTsFileName(tsId, suffix=None, ext=MRCS_EXT): return f'{tsId}_{suffix}.{ext}' if suffix else f'{tsId}.{ext}' - def getTmpOutFile(self, tsId, suffix=None, ext='mrc'): + def getTmpOutFile(self, tsId, suffix=None, ext=MRCS_EXT): return self._getTmpPath(tsId, self.getOutTsFileName(tsId, suffix=suffix, ext=ext)) - def getExtraOutFile(self, tsId, suffix=None, ext='mrc'): + def getExtraOutFile(self, tsId, suffix=None, ext=MRCS_EXT): return self._getExtraPath(tsId, self.getOutTsFileName(tsId, suffix=suffix, ext=ext)) def convertInputStep(self, tsObjId, generateAngleFile=True, imodInterpolation=True, doSwap=False, @@ -322,8 +316,8 @@ def genAlignmentFiles(self, ts, generateAngleFile=True, imodInterpolation=True, if oddEven: fnOdd = ts.getOddFileName() fnEven = ts.getEvenFileName() - outputOddTsFileName = self.getTmpOutFile(tsId, suffix=ODD, ext=MRCS_EXT) - outputEvenTsFileName = self.getTmpOutFile(tsId, suffix=EVEN, ext=MRCS_EXT) + outputOddTsFileName = self.getTmpOutFile(tsId, suffix=ODD) + outputEvenTsFileName = self.getTmpOutFile(tsId, suffix=EVEN) # Interpolation if imodInterpolation is None: diff --git a/imod/protocols/protocol_ctfCorrection.py b/imod/protocols/protocol_ctfCorrection.py index b69d26db..7557f3e8 100644 --- a/imod/protocols/protocol_ctfCorrection.py +++ b/imod/protocols/protocol_ctfCorrection.py @@ -238,13 +238,13 @@ def ctfCorrection(self, tsId): if self.applyToOddEven(ts): # ODD - paramsCtfPhaseFlip['inputStack'] = self.getTmpOutFile(tsId, suffix=ODD, ext=MRCS_EXT) - paramsCtfPhaseFlip['outputFileName'] = self.getExtraOutFile(tsId, suffix=ODD, ext=MRCS_EXT) + paramsCtfPhaseFlip['inputStack'] = self.getTmpOutFile(tsId, suffix=ODD) + paramsCtfPhaseFlip['outputFileName'] = self.getExtraOutFile(tsId, suffix=ODD) Plugin.runImod(self, 'ctfphaseflip', argsCtfPhaseFlip % paramsCtfPhaseFlip) # EVEN - paramsCtfPhaseFlip['inputStack'] = self.getTmpOutFile(tsId, suffix=EVEN, ext=MRCS_EXT) - paramsCtfPhaseFlip['outputFileName'] = self.getExtraOutFile(tsId, suffix=EVEN, ext=MRCS_EXT) + paramsCtfPhaseFlip['inputStack'] = self.getTmpOutFile(tsId, suffix=EVEN) + paramsCtfPhaseFlip['outputFileName'] = self.getExtraOutFile(tsId, suffix=EVEN) Plugin.runImod(self, 'ctfphaseflip', argsCtfPhaseFlip % paramsCtfPhaseFlip) def createOutputStep(self, tsId, presentAcqOrders): @@ -274,8 +274,8 @@ def createOutputStep(self, tsId, presentAcqOrders): newTi.setAcquisition(acq) newTi.setLocation(index + 1, self.getExtraOutFile(tsId)) if self.applyToOddEven(ts): - locationOdd = index + 1, self.getExtraOutFile(tsId, suffix=ODD, ext=MRCS_EXT) - locationEven = index + 1, self.getExtraOutFile(tsId, suffix=EVEN, ext=MRCS_EXT) + locationOdd = index + 1, self.getExtraOutFile(tsId, suffix=ODD) + locationEven = index + 1, self.getExtraOutFile(tsId, suffix=EVEN) newTi.setOddEven([ih.locationToXmipp(locationOdd), ih.locationToXmipp(locationEven)]) else: newTi.setOddEven([]) diff --git a/imod/protocols/protocol_doseFilter.py b/imod/protocols/protocol_doseFilter.py index 79545eea..e9505b48 100644 --- a/imod/protocols/protocol_doseFilter.py +++ b/imod/protocols/protocol_doseFilter.py @@ -165,12 +165,12 @@ def doseFilterStep(self, tsId): if self.applyToOddEven(ts): oddFn = firstItem.getOdd().split('@')[1] paramsMtffilter['input'] = oddFn - paramsMtffilter['output'] = self.getExtraOutFile(tsId, suffix=ODD, ext=MRCS_EXT) + paramsMtffilter['output'] = self.getExtraOutFile(tsId, suffix=ODD) Plugin.runImod(self, 'mtffilter', argsMtffilter % paramsMtffilter) evenFn = firstItem.getEven().split('@')[1] paramsMtffilter['input'] = evenFn - paramsMtffilter['output'] = self.getExtraOutFile(tsId, suffix=EVEN, ext=MRCS_EXT) + paramsMtffilter['output'] = self.getExtraOutFile(tsId, suffix=EVEN) Plugin.runImod(self, 'mtffilter', argsMtffilter % paramsMtffilter) def createOutputStep(self, tsId): @@ -188,8 +188,8 @@ def createOutputStep(self, tsId): newTi.copyInfo(tiltImage, copyId=True, copyTM=True) newTi.setAcquisition(tiltImage.getAcquisition()) if self.applyToOddEven(ts): - locationOdd = index + 1, self.getExtraOutFile(tsId, suffix=ODD, ext=MRCS_EXT) - locationEven = index + 1, self.getExtraOutFile(tsId, suffix=EVEN, ext=MRCS_EXT) + locationOdd = index + 1, self.getExtraOutFile(tsId, suffix=ODD) + locationEven = index + 1, self.getExtraOutFile(tsId, suffix=EVEN) newTi.setOddEven([ih.locationToXmipp(locationOdd), ih.locationToXmipp(locationEven)]) else: newTi.setOddEven([]) diff --git a/imod/protocols/protocol_tomoPreprocess.py b/imod/protocols/protocol_tomoPreprocess.py index d93138d2..795a4001 100644 --- a/imod/protocols/protocol_tomoPreprocess.py +++ b/imod/protocols/protocol_tomoPreprocess.py @@ -300,7 +300,7 @@ def generateOutputStackStep(self, tsId): if binning != 1: if runNewstack: - tmpPath = self.getTmpOutFile(tsId, ext=MRC_EXT) + tmpPath = self.getTmpOutFile(tsId) pwpath.moveFile(outputFile, tmpPath) inputTomoPath = tmpPath @@ -308,8 +308,8 @@ def generateOutputStackStep(self, tsId): pwpath.moveFile(oddEvenOutput[0], tmpPath) pwpath.moveFile(oddEvenOutput[1], tmpPath) inputTomoPath = tmpPath - inputOdd, inputEven = (self.getTmpOutFile(tsId, suffix=ODD, ext=MRC_EXT), - self.getTmpOutFile(tsId, suffix=EVEN, ext=MRC_EXT)) + inputOdd, inputEven = (self.getTmpOutFile(tsId, suffix=ODD), + self.getTmpOutFile(tsId, suffix=EVEN)) else: inputTomoPath = location if self.applyToOddEven(tomo): diff --git a/imod/protocols/protocol_tomoProjection.py b/imod/protocols/protocol_tomoProjection.py index 53f9717c..eaf1d659 100644 --- a/imod/protocols/protocol_tomoProjection.py +++ b/imod/protocols/protocol_tomoProjection.py @@ -24,18 +24,15 @@ # * # ***************************************************************************** -import os - from pwem.objects import Transform from pyworkflow import BETA from pyworkflow.object import Set import pyworkflow.protocol.params as params -import pyworkflow.utils.path as path from pwem.emlib.image import ImageHandler import tomo.objects as tomoObj from .. import Plugin -from .protocol_base import ProtImodBase, MRC_EXT +from .protocol_base import ProtImodBase, MRCS_EXT class ProtImodTomoProjection(ProtImodBase): @@ -116,7 +113,7 @@ def projectTomogram(self, tsId): paramsXYZproj = { 'input': tomo.getFileName(), - 'output': self.getExtraOutFile(tsId, ext=MRC_EXT), + 'output': self.getExtraOutFile(tsId, ext=MRCS_EXT), 'axis': self.getRotationAxis(), 'angles': str(self.minAngle.get()) + ',' + str(self.maxAngle.get()) + ',' + @@ -147,7 +144,7 @@ def generateOutputStackStep(self, tsId): newTi.setTsId(tsId) newTi.setAcquisitionOrder(index + 1) newTi.setLocation(index + 1, - self.getExtraOutFile(tsId, ext=MRC_EXT)) + self.getExtraOutFile(tsId, ext=MRCS_EXT)) newTi.setSamplingRate(sRate) newTs.append(newTi) diff --git a/imod/protocols/protocol_tomoReconstruction.py b/imod/protocols/protocol_tomoReconstruction.py index 583078c7..be5e826c 100644 --- a/imod/protocols/protocol_tomoReconstruction.py +++ b/imod/protocols/protocol_tomoReconstruction.py @@ -32,8 +32,7 @@ from tomo.objects import Tomogram from .. import Plugin -from .protocol_base import (ProtImodBase, EXT_MRC_ODD_NAME, EXT_MRC_EVEN_NAME, - EXT_MRCS_TS_EVEN_NAME, EXT_MRCS_TS_ODD_NAME, TLT_EXT, ODD, MRCS_EXT, EVEN, MRC_EXT, REC_EXT) +from .protocol_base import ProtImodBase, TLT_EXT, ODD, EVEN, MRC_EXT class ProtImodTomoReconstruction(ProtImodBase): @@ -312,12 +311,12 @@ def getArgs(): oddEvenTmp = [[], []] if self.applyToOddEven(ts): - paramsTilt['InputProjections'] = self.getTmpOutFile(tsId, suffix=ODD, ext=MRCS_EXT) + paramsTilt['InputProjections'] = self.getTmpOutFile(tsId, suffix=ODD) oddEvenTmp[0] = self.getExtraOutFile(tsId, suffix=ODD, ext=MRC_EXT) paramsTilt['OutputFile'] = oddEvenTmp[0] Plugin.runImod(self, 'tilt', argsTilt % paramsTilt) - paramsTilt['InputProjections'] = self.getTmpOutFile(tsId, suffix=EVEN, ext=MRCS_EXT) + paramsTilt['InputProjections'] = self.getTmpOutFile(tsId, suffix=EVEN) oddEvenTmp[1] = self.getExtraOutFile(tsId, suffix=EVEN, ext=MRC_EXT) paramsTilt['OutputFile'] = oddEvenTmp[1] Plugin.runImod(self, 'tilt', argsTilt % paramsTilt) diff --git a/imod/protocols/protocol_tsPreprocess.py b/imod/protocols/protocol_tsPreprocess.py index 9eb7fe53..7a98937b 100644 --- a/imod/protocols/protocol_tsPreprocess.py +++ b/imod/protocols/protocol_tsPreprocess.py @@ -309,10 +309,10 @@ def generateOutputStackStep(self, tsId): oddFn = firstItem.getOdd().split('@')[1] evenFn = firstItem.getEven().split('@')[1] paramsNewstack['input'] = oddFn - paramsNewstack['output'] = self.getExtraOutFile(tsId, suffix=ODD, ext=MRCS_EXT) + paramsNewstack['output'] = self.getExtraOutFile(tsId, suffix=ODD) Plugin.runImod(self, 'newstack', argsNewstack % paramsNewstack) paramsNewstack['input'] = evenFn - paramsNewstack['output'] = self.getExtraOutFile(tsId, suffix=EVEN, ext=MRCS_EXT) + paramsNewstack['output'] = self.getExtraOutFile(tsId, suffix=EVEN) Plugin.runImod(self, 'newstack', argsNewstack % paramsNewstack) output = self.getOutputSetOfTiltSeries(self.inputSetOfTiltSeries.get(), self.binning.get()) @@ -338,8 +338,8 @@ def generateOutputStackStep(self, tsId): newTi.setAcquisition(tiltImage.getAcquisition()) if self.applyToOddEven(ts): - locationOdd = index + 1, self.getExtraOutFile(tsId, suffix=ODD, ext=MRCS_EXT) - locationEven = index + 1, self.getExtraOutFile(tsId, suffix=EVEN, ext=MRCS_EXT) + locationOdd = index + 1, self.getExtraOutFile(tsId, suffix=ODD) + locationEven = index + 1, self.getExtraOutFile(tsId, suffix=EVEN) newTi.setOddEven([ih.locationToXmipp(locationOdd), ih.locationToXmipp(locationEven)]) else: newTi.setOddEven([]) diff --git a/imod/protocols/protocol_xRaysEraser.py b/imod/protocols/protocol_xRaysEraser.py index d21fb24f..2f8a3cc4 100644 --- a/imod/protocols/protocol_xRaysEraser.py +++ b/imod/protocols/protocol_xRaysEraser.py @@ -244,10 +244,10 @@ def eraseXraysStep(self, tsId): oddFn = firstItem.getOdd().split('@')[1] evenFn = firstItem.getEven().split('@')[1] paramsCcderaser['input'] = oddFn - paramsCcderaser['output'] = self.getExtraOutFile(tsId, suffix=ODD, ext=MRCS_EXT) + paramsCcderaser['output'] = self.getExtraOutFile(tsId, suffix=ODD) Plugin.runImod(self, 'ccderaser', argsCcderaser % paramsCcderaser) paramsCcderaser['input'] = evenFn - paramsCcderaser['output'] = self.getExtraOutFile(tsId, suffix=EVEN, ext=MRCS_EXT) + paramsCcderaser['output'] = self.getExtraOutFile(tsId, suffix=EVEN) Plugin.runImod(self, 'ccderaser', argsCcderaser % paramsCcderaser) def createOutputStep(self, tsId): @@ -267,8 +267,8 @@ def createOutputStep(self, tsId): newTi.setLocation(index + 1, self.getExtraOutFile(tsId)) if self.applyToOddEven(ts): - locationOdd = index + 1, self.getExtraOutFile(tsId, suffix=ODD, ext=MRCS_EXT) - locationEven = index + 1, self.getExtraOutFile(tsId, suffix=EVEN, ext=MRCS_EXT) + locationOdd = index + 1, self.getExtraOutFile(tsId, suffix=ODD) + locationEven = index + 1, self.getExtraOutFile(tsId, suffix=EVEN) newTi.setOddEven([ih.locationToXmipp(locationOdd), ih.locationToXmipp(locationEven)]) else: newTi.setOddEven([]) diff --git a/imod/tests/test_protocols_imod.py b/imod/tests/test_protocols_imod.py index 46ef956d..654a2e03 100644 --- a/imod/tests/test_protocols_imod.py +++ b/imod/tests/test_protocols_imod.py @@ -460,7 +460,7 @@ def test_doseFilterOutputTS(self): tsId = ts.getFirstItem().getTsId() self.assertTrue(os.path.exists(os.path.join(self.protDoseFilter._getExtraPath(tsId), - tsId + ".mrc"))) + tsId + ".mrcs"))) def test_xRaysEraserOutputTS(self): @@ -470,7 +470,7 @@ def test_xRaysEraserOutputTS(self): tsId = ts.getFirstItem().getTsId() self.assertTrue(os.path.exists(os.path.join(self.protXRaysEraser._getExtraPath(tsId), - tsId + ".mrc"))) + tsId + ".mrcs"))) def test_excludeViewsOutputTS(self): @@ -480,7 +480,7 @@ def test_excludeViewsOutputTS(self): tsId = ts.getFirstItem().getTsId() self.assertTrue(os.path.exists(os.path.join(self.protExcludeViews._getExtraPath(tsId), - tsId + ".mrc"))) + tsId + ".mrcs"))) for index, tsOut in enumerate(ts): self.assertEqual(tsOut.getSize(), self.excludeViewsOutputSizes[tsOut.getTsId()]) @@ -493,7 +493,7 @@ def test_normalizationOutputTS(self): tsId = ts.getFirstItem().getTsId() self.assertTrue(os.path.exists(os.path.join(self.protTSNormalization._getExtraPath(tsId), - tsId + ".mrc"))) + tsId + ".mrcs"))) inSamplingRate = self.protTSNormalization.inputSetOfTiltSeries.get().getSamplingRate() outSamplingRate = ts.getSamplingRate() @@ -507,7 +507,7 @@ def test_prealignmentOutputTS(self): tsId = ts.getFirstItem().getTsId() outputLocation = os.path.join(self.protXcorr._getExtraPath(tsId), - tsId + ".mrc") + tsId + ".mrcs") self.assertTrue(os.path.exists(outputLocation)) @@ -521,7 +521,7 @@ def test_prealignmentOutputInterpolatedTS(self): tsId = ts.getFirstItem().getTsId() outputLocation = os.path.join(self.protXcorr._getExtraPath(tsId), - tsId + ".mrc") + tsId + ".mrcs") self.assertTrue(os.path.exists(outputLocation)) @@ -552,7 +552,7 @@ def test_fiducialAlignmentOutputTS(self): tsId = output.getFirstItem().getTsId() outputLocation = os.path.join(self.protFiducialAlignment._getExtraPath(tsId), - tsId + ".mrc") + tsId + ".mrcs") self.assertTrue(os.path.exists(outputLocation)) @@ -565,7 +565,7 @@ def test_fiducialAlignmentOutputTS(self): tsId = output.getFirstItem().getTsId() outputLocation = os.path.join(self.protFiducialAlignment._getExtraPath(tsId), - tsId + ".mrc") + tsId + ".mrcs") self.assertTrue(os.path.exists(outputLocation)) @@ -608,7 +608,7 @@ def test_applyTransformationMatrixOutputInterpolatedTS(self): tsId = output.getFirstItem().getTsId() outputLocation = os.path.join(self.protApplyTransformationMatrix._getExtraPath(tsId), - tsId + ".mrc") + tsId + ".mrcs") self.assertTrue(os.path.exists(outputLocation)) @@ -765,7 +765,7 @@ def test_ctfCorrectionOutput(self): for ts in output: tsId = ts.getTsId() outputLocation = os.path.join(self.protCTFCorrection._getExtraPath(tsId), - '%s.mrc' % tsId) + '%s.mrcs' % tsId) self.assertTrue(os.path.exists(outputLocation)) From 38057cca4e19091737084cddcf3f366dd757f40a Mon Sep 17 00:00:00 2001 From: Grigory Sharov Date: Thu, 30 May 2024 12:53:41 +0100 Subject: [PATCH 39/49] pep8 cleanup, remove unused imports --- .../protocol_applyTransformationMatrix.py | 2 +- imod/protocols/protocol_base.py | 11 ++-- imod/protocols/protocol_ctfCorrection.py | 3 +- imod/protocols/protocol_doseFilter.py | 2 +- imod/protocols/protocol_fiducialAlignment.py | 8 +-- imod/protocols/protocol_fiducialModel.py | 3 - imod/protocols/protocol_goldBeadPicker3d.py | 3 - imod/protocols/protocol_tomoPreprocess.py | 2 +- imod/protocols/protocol_tomoReconstruction.py | 61 +++++++++---------- imod/protocols/protocol_tsPreprocess.py | 2 +- imod/protocols/protocol_xCorrPrealignment.py | 6 +- imod/protocols/protocol_xRaysEraser.py | 6 +- 12 files changed, 47 insertions(+), 62 deletions(-) diff --git a/imod/protocols/protocol_applyTransformationMatrix.py b/imod/protocols/protocol_applyTransformationMatrix.py index da2f5b98..dbc4db82 100644 --- a/imod/protocols/protocol_applyTransformationMatrix.py +++ b/imod/protocols/protocol_applyTransformationMatrix.py @@ -31,7 +31,7 @@ from pwem.emlib.image import ImageHandler from tomo.objects import TiltSeries, TiltImage from .. import Plugin, utils -from .protocol_base import ProtImodBase,XF_EXT, ODD, EVEN, MRCS_EXT +from .protocol_base import ProtImodBase, XF_EXT, ODD, EVEN class ProtImodApplyTransformationMatrix(ProtImodBase): diff --git a/imod/protocols/protocol_base.py b/imod/protocols/protocol_base.py index 9f130ded..507465b4 100644 --- a/imod/protocols/protocol_base.py +++ b/imod/protocols/protocol_base.py @@ -24,7 +24,6 @@ # * # ***************************************************************************** import logging -import os from pyworkflow.object import Set, CsvList, Pointer from pyworkflow.protocol import STEPS_PARALLEL, params @@ -451,7 +450,7 @@ def getOutputSetOfTiltSeries(self, inputSet, binning=1) -> SetOfTiltSeries: if binning > 1: samplingRate = inputSet.getSamplingRate() - samplingRate *= self.binning.get() + samplingRate *= self.binning outputSetOfTiltSeries.setSamplingRate(samplingRate) outputSetOfTiltSeries.setStreamState(Set.STREAM_OPEN) @@ -481,7 +480,7 @@ def getOutputInterpolatedSetOfTiltSeries(self, inputSet): if self.binning > 1: samplingRate = inputSet.getSamplingRate() - samplingRate *= self.binning.get() + samplingRate *= self.binning outputInterpolatedSetOfTiltSeries.setSamplingRate(samplingRate) outputInterpolatedSetOfTiltSeries.setStreamState(Set.STREAM_OPEN) @@ -598,8 +597,6 @@ def getOutputSetOfTomograms(self, inputSet, binning=1): if self.Tomograms: getattr(self, OUTPUT_TOMOGRAMS_NAME).enableAppend() - - else: outputSetOfTomograms = self._createSetOfTomograms() @@ -612,7 +609,7 @@ def getOutputSetOfTomograms(self, inputSet, binning=1): if binning > 1: samplingRate = inputSet.getSamplingRate() - samplingRate *= self.binning.get() + samplingRate *= self.binning outputSetOfTomograms.setSamplingRate(samplingRate) outputSetOfTomograms.setStreamState(Set.STREAM_OPEN) @@ -759,7 +756,7 @@ def createOutputFailedSet(self, ts, presentAcqOrders=None): newTi.setAcquisition(tiltImage.getAcquisition()) newTi.setLocation(tiltImage.getLocation()) if hasattr(self, "binning") and self.binning > 1: - newTi.setSamplingRate(tiltImage.getSamplingRate() * self.binning.get()) + newTi.setSamplingRate(tiltImage.getSamplingRate() * self.binning) newTs.append(newTi) ih = ImageHandler() diff --git a/imod/protocols/protocol_ctfCorrection.py b/imod/protocols/protocol_ctfCorrection.py index 7557f3e8..9c719158 100644 --- a/imod/protocols/protocol_ctfCorrection.py +++ b/imod/protocols/protocol_ctfCorrection.py @@ -31,7 +31,7 @@ from tomo.objects import TiltSeries, TiltImage from tomo.utils import getCommonTsAndCtfElements from .. import Plugin, utils -from .protocol_base import ProtImodBase, DEFOCUS_EXT, TLT_EXT, XF_EXT, ODD, MRCS_EXT, EVEN +from .protocol_base import ProtImodBase, DEFOCUS_EXT, TLT_EXT, XF_EXT, ODD, EVEN class ProtImodCtfCorrection(ProtImodBase): @@ -251,7 +251,6 @@ def createOutputStep(self, tsId, presentAcqOrders): if tsId not in self._failedTs: inTsSet = self.inputSetOfTiltSeries.get() outputSetOfTs = self.getOutputSetOfTiltSeries(inTsSet) - extraPrefix = self._getExtraPath(tsId) newTs = TiltSeries(tsId=tsId) ts = self.tsDict[tsId] diff --git a/imod/protocols/protocol_doseFilter.py b/imod/protocols/protocol_doseFilter.py index e9505b48..29e93a41 100644 --- a/imod/protocols/protocol_doseFilter.py +++ b/imod/protocols/protocol_doseFilter.py @@ -30,7 +30,7 @@ from pwem.emlib.image import ImageHandler from .. import Plugin, utils -from .protocol_base import ProtImodBase, ODD, MRCS_EXT, EVEN +from .protocol_base import ProtImodBase, ODD, EVEN SCIPION_IMPORT = 0 FIXED_DOSE = 1 diff --git a/imod/protocols/protocol_fiducialAlignment.py b/imod/protocols/protocol_fiducialAlignment.py index ee521989..473eb15f 100644 --- a/imod/protocols/protocol_fiducialAlignment.py +++ b/imod/protocols/protocol_fiducialAlignment.py @@ -200,11 +200,11 @@ def _defineParams(self, form): label='Rotation solution type', display=params.EnumParam.DISPLAY_HLIST, help='Type of rotation solution: See rotOption in tiltalign IMOD command \n' - '* No rotation: The in-plane rotation will not be estimated\n' - '* One rotation: To solve for a single rotation variable \n' - '* Group rotations: Group views to solve for fewer rotations variables. Automapping of ' + '* No rotation: The in-plane rotation will not be estimated\n' + '* One rotation: To solve for a single rotation variable \n' + '* Group rotations: Group views to solve for fewer rotations variables. Automapping of ' 'rotation variables linearly changing values\n' - '* Solve for all rotations: for each view having an independent rotation\n') + '* Solve for all rotations: for each view having an independent rotation\n') form.addParam('groupRotationSize', params.IntParam, diff --git a/imod/protocols/protocol_fiducialModel.py b/imod/protocols/protocol_fiducialModel.py index 3e53ad52..749c3b69 100644 --- a/imod/protocols/protocol_fiducialModel.py +++ b/imod/protocols/protocol_fiducialModel.py @@ -370,8 +370,6 @@ def generateFiducialModelStep(self, tsId): Plugin.runImod(self, 'beadtrack', argsBeadtrack % paramsBeadtrack) def translateFiducialPointModelStep(self, tsId): - ts = self.tsDict[tsId] - # Check that previous steps have been completed satisfactorily gapsFidFile = self.getExtraOutFile(tsId, suffix='gaps', ext=FID_EXT) if os.path.exists(gapsFidFile): @@ -540,7 +538,6 @@ def createOutputFailedSetStep(self, tsId): def translateTrackCom(self, ts, paramsDict): tsId = ts.getTsId() extraPrefix = self._getExtraPath(tsId) - tmpPrefix = self._getTmpPath(tsId) firstItem = ts.getFirstItem() trackFilePath = os.path.join(extraPrefix, diff --git a/imod/protocols/protocol_goldBeadPicker3d.py b/imod/protocols/protocol_goldBeadPicker3d.py index 0c727160..64ff9556 100644 --- a/imod/protocols/protocol_goldBeadPicker3d.py +++ b/imod/protocols/protocol_goldBeadPicker3d.py @@ -24,11 +24,8 @@ # * # ***************************************************************************** -import os - from pyworkflow import BETA import pyworkflow.protocol.params as params -from pyworkflow.utils import path, removeBaseExt from pyworkflow.object import Set import tomo.objects as tomoObj import tomo.constants as constants diff --git a/imod/protocols/protocol_tomoPreprocess.py b/imod/protocols/protocol_tomoPreprocess.py index 795a4001..9946a9b5 100644 --- a/imod/protocols/protocol_tomoPreprocess.py +++ b/imod/protocols/protocol_tomoPreprocess.py @@ -29,7 +29,7 @@ import pyworkflow.utils.path as pwpath from tomo.objects import Tomogram, SetOfTomograms from .. import Plugin -from .protocol_base import ProtImodBase,OUTPUT_TOMOGRAMS_NAME, MRC_EXT, ODD, EVEN +from .protocol_base import ProtImodBase, OUTPUT_TOMOGRAMS_NAME, MRC_EXT, ODD, EVEN class ProtImodTomoNormalization(ProtImodBase): diff --git a/imod/protocols/protocol_tomoReconstruction.py b/imod/protocols/protocol_tomoReconstruction.py index be5e826c..7bbd8e18 100644 --- a/imod/protocols/protocol_tomoReconstruction.py +++ b/imod/protocols/protocol_tomoReconstruction.py @@ -88,25 +88,25 @@ def _defineParams(self, form): help='Number of pixels to cut out in X, centered on the middle in X. Leave 0 for default X.') lineShift = form.addLine('Tomogram shift (Å)', - expertLevel=params.LEVEL_ADVANCED, - help="This entry allows one to shift the reconstructed" - " slice in X or Z before it is output. If the " - " X shift is positive, the slice will be shifted to " - " the right, and the output will contain the left " - " part of the whole potentially reconstructable area. " - " If the Z shift is positive, the slice is shifted " - " upward. The Z entry is optional and defaults to 0 when " - " omitted.") + expertLevel=params.LEVEL_ADVANCED, + help="This entry allows one to shift the reconstructed" + " slice in X or Z before it is output. If the " + " X shift is positive, the slice will be shifted to " + " the right, and the output will contain the left " + " part of the whole potentially reconstructable area. " + " If the Z shift is positive, the slice is shifted " + " upward. The Z entry is optional and defaults to 0 when " + " omitted.") lineShift.addParam('tomoShiftX', - params.FloatParam, - default=0, - label=' in X ') + params.FloatParam, + default=0, + label=' in X ') lineShift.addParam('tomoShiftZ', - params.FloatParam, - default=0, - label=' in Z ') + params.FloatParam, + default=0, + label=' in Z ') lineoffSet = form.addLine('Offset (deg) of the ', expertLevel=params.LEVEL_ADVANCED, @@ -116,25 +116,22 @@ def _defineParams(self, form): "projection images, cutting the X-axis at NX/2 + offset instead of NX/2.") lineoffSet.addParam('angleOffset', - params.FloatParam, - default=0, - label='Tilt angles ', - help='Apply an angle offset in degrees to all tilt ' - 'angles. This offset positively rotates the ' - 'reconstructed sections anticlockwise.') + params.FloatParam, + default=0, + label='Tilt angles ', + help='Apply an angle offset in degrees to all tilt ' + 'angles. This offset positively rotates the ' + 'reconstructed sections anticlockwise.') lineoffSet.addParam('tiltAxisOffset', - params.FloatParam, - default=0, - label='Tilt axis', - help='Apply an offset to the tilt axis in a stack of ' - 'full-sized projection images, cutting the ' - 'X-axis at NX/2 + offset instead of NX/2. The ' - 'DELXX entry is optional and defaults to 0 ' - 'when omitted.') - - - + params.FloatParam, + default=0, + label='Tilt axis', + help='Apply an offset to the tilt axis in a stack of ' + 'full-sized projection images, cutting the ' + 'X-axis at NX/2 + offset instead of NX/2. The ' + 'DELXX entry is optional and defaults to 0 ' + 'when omitted.') form.addParam('superSampleFactor', params.IntParam, diff --git a/imod/protocols/protocol_tsPreprocess.py b/imod/protocols/protocol_tsPreprocess.py index 7a98937b..5b916c79 100644 --- a/imod/protocols/protocol_tsPreprocess.py +++ b/imod/protocols/protocol_tsPreprocess.py @@ -29,7 +29,7 @@ from pwem.emlib.image import ImageHandler import tomo.objects as tomoObj from .. import Plugin -from .protocol_base import ProtImodBase, OUTPUT_TILTSERIES_NAME, XF_EXT, ODD, EVEN, MRCS_EXT +from .protocol_base import ProtImodBase, OUTPUT_TILTSERIES_NAME, XF_EXT, ODD, EVEN from ..utils import genXfFile diff --git a/imod/protocols/protocol_xCorrPrealignment.py b/imod/protocols/protocol_xCorrPrealignment.py index 6c4e07d1..8376c803 100644 --- a/imod/protocols/protocol_xCorrPrealignment.py +++ b/imod/protocols/protocol_xCorrPrealignment.py @@ -24,7 +24,6 @@ # * # ***************************************************************************** -import os import numpy as np from pyworkflow import BETA @@ -125,7 +124,8 @@ def _defineParams(self, form): trimming = form.addGroup('Trimming parameters', expertLevel=params.LEVEL_ADVANCED) - self.trimimgForm(trimming, pxTrimCondition='False', correlationCondition='True', levelType=params.LEVEL_ADVANCED) + self.trimimgForm(trimming, pxTrimCondition='False', + correlationCondition='True', levelType=params.LEVEL_ADVANCED) self.filteringParametersForm(form, condition='True', levelType=params.LEVEL_ADVANCED) # -------------------------- INSERT steps functions ----------------------- @@ -134,7 +134,7 @@ def _insertAllSteps(self): for tsId in self.tsDict.keys(): self._insertFunctionStep(self.convertInputStep, tsId) self._insertFunctionStep(self.computeXcorrStep, tsId) - self._insertFunctionStep(self.generateOutputStackStep,tsId) + self._insertFunctionStep(self.generateOutputStackStep, tsId) if self.computeAlignment.get() == 0: self._insertFunctionStep(self.computeInterpolatedStackStep, tsId) self._insertFunctionStep(self.closeOutputSetsStep) diff --git a/imod/protocols/protocol_xRaysEraser.py b/imod/protocols/protocol_xRaysEraser.py index 2f8a3cc4..59005dc4 100644 --- a/imod/protocols/protocol_xRaysEraser.py +++ b/imod/protocols/protocol_xRaysEraser.py @@ -24,8 +24,6 @@ # * # ***************************************************************************** -import os - from pyworkflow import BETA import pyworkflow.protocol.params as params from pyworkflow.object import Set @@ -33,8 +31,8 @@ import tomo.objects as tomoObj from .. import Plugin -from .protocol_base import ProtImodBase, OUTPUT_TILTSERIES_NAME, EXT_MRCS_TS_ODD_NAME, EXT_MRCS_TS_EVEN_NAME, ODD, \ - MRCS_EXT, EVEN, MOD_EXT +from .protocol_base import (ProtImodBase, OUTPUT_TILTSERIES_NAME, + ODD, EVEN, MOD_EXT) class ProtImodXraysEraser(ProtImodBase): From a9647b6ab74f1cbc00b9da61bcf213db8cae0839 Mon Sep 17 00:00:00 2001 From: Grigory Sharov Date: Thu, 30 May 2024 12:56:23 +0100 Subject: [PATCH 40/49] fix failing test --- imod/tests/test_protocols_imod.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imod/tests/test_protocols_imod.py b/imod/tests/test_protocols_imod.py index 654a2e03..943045f6 100644 --- a/imod/tests/test_protocols_imod.py +++ b/imod/tests/test_protocols_imod.py @@ -670,7 +670,7 @@ def test_tomoProjectionOutputTiltSeriesSize(self): ih = ImageHandler() outputDimensions = ih.getDimensions(output.getFirstItem().getFirstItem().getFileName()) - self.assertEqual(outputDimensions, (256, 256, 61, 1)) + self.assertEqual(outputDimensions, (256, 256, 1, 61)) # Sampling rate inSamplingRate = self.protTomoProjection.inputSetOfTomograms.get().getSamplingRate() From 946a78d8c363f3136e0e61a58a368378b02ce942 Mon Sep 17 00:00:00 2001 From: Grigory Sharov Date: Thu, 30 May 2024 13:00:07 +0100 Subject: [PATCH 41/49] fix multiplication --- imod/protocols/protocol_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imod/protocols/protocol_base.py b/imod/protocols/protocol_base.py index 507465b4..d7fa53dd 100644 --- a/imod/protocols/protocol_base.py +++ b/imod/protocols/protocol_base.py @@ -756,7 +756,7 @@ def createOutputFailedSet(self, ts, presentAcqOrders=None): newTi.setAcquisition(tiltImage.getAcquisition()) newTi.setLocation(tiltImage.getLocation()) if hasattr(self, "binning") and self.binning > 1: - newTi.setSamplingRate(tiltImage.getSamplingRate() * self.binning) + newTi.setSamplingRate(tiltImage.getSamplingRate() * self.binning.get()) newTs.append(newTi) ih = ImageHandler() From 6773557563df903c7f867d1fbd410cbc656acb1b Mon Sep 17 00:00:00 2001 From: Grigory Sharov Date: Thu, 30 May 2024 13:02:20 +0100 Subject: [PATCH 42/49] fix multiplication --- imod/protocols/protocol_base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/imod/protocols/protocol_base.py b/imod/protocols/protocol_base.py index d7fa53dd..099a5b07 100644 --- a/imod/protocols/protocol_base.py +++ b/imod/protocols/protocol_base.py @@ -450,7 +450,7 @@ def getOutputSetOfTiltSeries(self, inputSet, binning=1) -> SetOfTiltSeries: if binning > 1: samplingRate = inputSet.getSamplingRate() - samplingRate *= self.binning + samplingRate *= self.binning.get() outputSetOfTiltSeries.setSamplingRate(samplingRate) outputSetOfTiltSeries.setStreamState(Set.STREAM_OPEN) @@ -480,7 +480,7 @@ def getOutputInterpolatedSetOfTiltSeries(self, inputSet): if self.binning > 1: samplingRate = inputSet.getSamplingRate() - samplingRate *= self.binning + samplingRate *= self.binning.get() outputInterpolatedSetOfTiltSeries.setSamplingRate(samplingRate) outputInterpolatedSetOfTiltSeries.setStreamState(Set.STREAM_OPEN) @@ -609,7 +609,7 @@ def getOutputSetOfTomograms(self, inputSet, binning=1): if binning > 1: samplingRate = inputSet.getSamplingRate() - samplingRate *= self.binning + samplingRate *= self.binning.get() outputSetOfTomograms.setSamplingRate(samplingRate) outputSetOfTomograms.setStreamState(Set.STREAM_OPEN) From f9957b576b3161d09fc388963c6a40605c4e4fce Mon Sep 17 00:00:00 2001 From: JorMaister Date: Wed, 5 Jun 2024 16:21:58 +0200 Subject: [PATCH 43/49] start this, not finished --- imod/protocols/protocol_base.py | 45 ++++++++++++++++++------ imod/protocols/protocol_ctfCorrection.py | 14 +++++++- 2 files changed, 48 insertions(+), 11 deletions(-) diff --git a/imod/protocols/protocol_base.py b/imod/protocols/protocol_base.py index 099a5b07..8e2222b9 100644 --- a/imod/protocols/protocol_base.py +++ b/imod/protocols/protocol_base.py @@ -280,7 +280,7 @@ def convertInputStep(self, tsObjId, generateAngleFile=True, imodInterpolation=Tr self.genAlignmentFiles(ts, generateAngleFile=generateAngleFile, imodInterpolation=imodInterpolation, doSwap=doSwap, oddEven=oddEven, presentAcqOrders=presentAcqOrders) - def applyNewStackBasic(self, ts, outputTsFileName, inputTsFileName, xfFile, doSwap, tsExcludedIndices=None): + def applyNewStackBasic(self, ts, outputTsFileName, inputTsFileName, xfFile=None, doSwap=None, tsExcludedIndices=None): argsAlignment, paramsAlignment = self.getBasicNewstackParams(ts, outputTsFileName, inputTsFileName=inputTsFileName, @@ -324,7 +324,7 @@ def genAlignmentFiles(self, ts, generateAngleFile=True, imodInterpolation=True, path.createLink(inTsFileName, outputTsFileName) elif imodInterpolation: - logger.info("Apply the transformation form the input tilt-series") + logger.info("Apply the transformation from the input tilt-series") # Use IMOD newstack interpolation if firstTi.hasTransform(): @@ -338,12 +338,18 @@ def genAlignmentFiles(self, ts, generateAngleFile=True, imodInterpolation=True, tsExcludedIndices = [ti.getIndex() for ti in ts if not ti.getAcquisitionOrder() in presentAcqOrders] else: tsExcludedIndices = [ti.getIndex() for ti in ts if not ti.getAcquisitionOrder()] - self.applyNewStackBasic(ts, outputTsFileName, inTsFileName, xfFile, doSwap, + self.applyNewStackBasic(ts, outputTsFileName, inTsFileName, + xfFile=xfFile, + doSwap=doSwap, tsExcludedIndices=tsExcludedIndices) if oddEven: - self.applyNewStackBasic(ts, outputOddTsFileName, fnOdd, xfFile, doSwap, + self.applyNewStackBasic(ts, outputOddTsFileName, fnOdd, + xfFile=xfFile, + doSwap=doSwap, tsExcludedIndices=tsExcludedIndices) - self.applyNewStackBasic(ts, outputEvenTsFileName, fnEven, xfFile, doSwap, + self.applyNewStackBasic(ts, outputEvenTsFileName, fnEven, + xfFile=xfFile, + doSwap=doSwap, tsExcludedIndices=tsExcludedIndices) # If some views were excluded to generate the new stack, a new xfFile containing them should be @@ -353,12 +359,31 @@ def genAlignmentFiles(self, ts, generateAngleFile=True, imodInterpolation=True, else: # The given TS is interpolated - logger.info("Tilt-series linked [%s]" % tsId) - path.createLink(firstTi.getFileName(), outputTsFileName) + logger.info("The given TS is interpolated [%s]" % tsId) + if presentAcqOrders: + logger.info("Tilt-series re-stacked with IMOD") + if len(presentAcqOrders) != len(ts): + tsExcludedIndices = [ti.getIndex() for ti in ts if not ti.getAcquisitionOrder() in presentAcqOrders] + self.applyNewStackBasic(ts, outputTsFileName, inTsFileName, + doSwap=doSwap, + tsExcludedIndices=tsExcludedIndices) + # TODO: Ask Vilas about this + # if oddEven: + # self.applyNewStackBasic(ts, outputOddTsFileName, fnOdd, + # xfFile=xfFile, + # doSwap=doSwap, + # tsExcludedIndices=tsExcludedIndices) + # self.applyNewStackBasic(ts, outputEvenTsFileName, fnEven, + # xfFile=xfFile, + # doSwap=doSwap, + # tsExcludedIndices=tsExcludedIndices) + else: + logger.info("Tilt-series linked [%s]" % tsId) + path.createLink(firstTi.getFileName(), outputTsFileName) - if oddEven: - path.createLink(fnOdd, outputOddTsFileName) - path.createLink(fnEven, outputEvenTsFileName) + if oddEven: + path.createLink(fnOdd, outputOddTsFileName) + path.createLink(fnEven, outputEvenTsFileName) # Use Xmipp interpolation via Scipion else: diff --git a/imod/protocols/protocol_ctfCorrection.py b/imod/protocols/protocol_ctfCorrection.py index 9c719158..bff1ec5f 100644 --- a/imod/protocols/protocol_ctfCorrection.py +++ b/imod/protocols/protocol_ctfCorrection.py @@ -164,6 +164,18 @@ def _defineParams(self, form): # -------------------------- INSERT steps functions ----------------------- def _insertAllSteps(self): + # JORGE + import os + fname = "/home/jjimenez/test_JJ.txt" + if os.path.exists(fname): + os.remove(fname) + fjj = open(fname, "a+") + fjj.write('JORGE--------->onDebugMode PID {}'.format(os.getpid())) + fjj.close() + print('JORGE--------->onDebugMode PID {}'.format(os.getpid())) + import time + time.sleep(10) + # JORGE_END nonMatchingTsIds = [] self._initialize() for tsId in self.tsDict.keys(): # Stores the steps serializing the tsId instead of the whole ts object @@ -196,7 +208,7 @@ def convertInputsStep(self, tsId, presentAcqOrders): # Generate the defocus file self.generateDefocusFile(tsId, presentAcqOrders=presentAcqOrders) - @ProtImodBase.tryExceptDecorator + # @ProtImodBase.tryExceptDecorator def ctfCorrection(self, tsId): ts = self.tsDict[tsId] acq = ts.getAcquisition() From cd519c4d1330b1987409ad5e6bfe0b4860d9997b Mon Sep 17 00:00:00 2001 From: JorMaister Date: Thu, 6 Jun 2024 12:13:46 +0200 Subject: [PATCH 44/49] Ut can deal with excluded views in the CTF and/or in the TS when entering with interpolated or not aligned tilt-series. --- CHANGES.txt | 1 + imod/protocols/protocol_base.py | 26 ++++++++++-------------- imod/protocols/protocol_ctfCorrection.py | 14 +------------ 3 files changed, 13 insertions(+), 28 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index bd1bff9b..c2cb802a 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -7,6 +7,7 @@ - The order of the parameters in the forms was slightly modified according to the user actions. - Protocol CTF correction: * Now it can deal with excluded views in the CTF and/or in the TS. + * The same when entering with interpolated or not aligned TS. * The xf, defocus, and tlt files are generated inside each TS extra sub-directory. Developers: - Update the acquisition order in the CTFTomo objects (field added to that class in scipion-em-tomo v3.7.0). diff --git a/imod/protocols/protocol_base.py b/imod/protocols/protocol_base.py index 8e2222b9..a46e0bf4 100644 --- a/imod/protocols/protocol_base.py +++ b/imod/protocols/protocol_base.py @@ -324,17 +324,16 @@ def genAlignmentFiles(self, ts, generateAngleFile=True, imodInterpolation=True, path.createLink(inTsFileName, outputTsFileName) elif imodInterpolation: - logger.info("Apply the transformation from the input tilt-series") - + xfFile = self.getExtraOutFile(tsId, ext=XF_EXT) # Use IMOD newstack interpolation if firstTi.hasTransform(): # Generate transformation matrices file (xf) - xfFile = self.getExtraOutFile(tsId, ext=XF_EXT) utils.genXfFile(ts, xfFile) # Generate the interpolated TS with IMOD's newstack program - logger.info("Tilt-series interpolated with IMOD [%s]" % tsId) + logger.info("Tilt-series will be interpolated with IMOD [%s]" % tsId) if presentAcqOrders: + logger.info("Tilt-series re-stacked with IMOD") tsExcludedIndices = [ti.getIndex() for ti in ts if not ti.getAcquisitionOrder() in presentAcqOrders] else: tsExcludedIndices = [ti.getIndex() for ti in ts if not ti.getAcquisitionOrder()] @@ -359,7 +358,7 @@ def genAlignmentFiles(self, ts, generateAngleFile=True, imodInterpolation=True, else: # The given TS is interpolated - logger.info("The given TS is interpolated [%s]" % tsId) + logger.info("The given TS is interpolated or not aligned [%s]" % tsId) if presentAcqOrders: logger.info("Tilt-series re-stacked with IMOD") if len(presentAcqOrders) != len(ts): @@ -367,16 +366,13 @@ def genAlignmentFiles(self, ts, generateAngleFile=True, imodInterpolation=True, self.applyNewStackBasic(ts, outputTsFileName, inTsFileName, doSwap=doSwap, tsExcludedIndices=tsExcludedIndices) - # TODO: Ask Vilas about this - # if oddEven: - # self.applyNewStackBasic(ts, outputOddTsFileName, fnOdd, - # xfFile=xfFile, - # doSwap=doSwap, - # tsExcludedIndices=tsExcludedIndices) - # self.applyNewStackBasic(ts, outputEvenTsFileName, fnEven, - # xfFile=xfFile, - # doSwap=doSwap, - # tsExcludedIndices=tsExcludedIndices) + if oddEven: + self.applyNewStackBasic(ts, outputOddTsFileName, fnOdd, + doSwap=doSwap, + tsExcludedIndices=tsExcludedIndices) + self.applyNewStackBasic(ts, outputEvenTsFileName, fnEven, + doSwap=doSwap, + tsExcludedIndices=tsExcludedIndices) else: logger.info("Tilt-series linked [%s]" % tsId) path.createLink(firstTi.getFileName(), outputTsFileName) diff --git a/imod/protocols/protocol_ctfCorrection.py b/imod/protocols/protocol_ctfCorrection.py index bff1ec5f..9c719158 100644 --- a/imod/protocols/protocol_ctfCorrection.py +++ b/imod/protocols/protocol_ctfCorrection.py @@ -164,18 +164,6 @@ def _defineParams(self, form): # -------------------------- INSERT steps functions ----------------------- def _insertAllSteps(self): - # JORGE - import os - fname = "/home/jjimenez/test_JJ.txt" - if os.path.exists(fname): - os.remove(fname) - fjj = open(fname, "a+") - fjj.write('JORGE--------->onDebugMode PID {}'.format(os.getpid())) - fjj.close() - print('JORGE--------->onDebugMode PID {}'.format(os.getpid())) - import time - time.sleep(10) - # JORGE_END nonMatchingTsIds = [] self._initialize() for tsId in self.tsDict.keys(): # Stores the steps serializing the tsId instead of the whole ts object @@ -208,7 +196,7 @@ def convertInputsStep(self, tsId, presentAcqOrders): # Generate the defocus file self.generateDefocusFile(tsId, presentAcqOrders=presentAcqOrders) - # @ProtImodBase.tryExceptDecorator + @ProtImodBase.tryExceptDecorator def ctfCorrection(self, tsId): ts = self.tsDict[tsId] acq = ts.getAcquisition() From 9b720486c8446c2c95b9ab57c6fcce735043de98 Mon Sep 17 00:00:00 2001 From: Grigory Sharov Date: Sat, 8 Jun 2024 17:29:09 +0100 Subject: [PATCH 45/49] fix missing protocol --- imod/protocols.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imod/protocols.conf b/imod/protocols.conf index eab5fd0a..4124f730 100644 --- a/imod/protocols.conf +++ b/imod/protocols.conf @@ -5,7 +5,7 @@ Tomography = [ ]}, {"tag": "section", "text": "Tilt-series", "children": [ {"tag": "protocol_group", "text": "Tilt-series preprocess", "openItem": "False", "children": [ - {"tag": "protocol", "value": "ProtImodTSNormalization", "text": "default"}, + {"tag": "protocol", "value": "ProtImodTsNormalization", "text": "default"}, {"tag": "protocol", "value": "ProtImodXraysEraser", "text": "default"}, {"tag": "protocol", "value": "ProtImodDoseFilter", "text": "default"}, {"tag": "protocol", "value": "ProtImodExcludeViews", "text": "default"} From a67ea30859a112b58ddc5951ff5794af5e0ab601 Mon Sep 17 00:00:00 2001 From: Grigory Sharov Date: Sat, 8 Jun 2024 17:29:29 +0100 Subject: [PATCH 46/49] fix parsing empty strings --- imod/protocols/protocol_ctfEstimation_automatic.py | 1 + 1 file changed, 1 insertion(+) diff --git a/imod/protocols/protocol_ctfEstimation_automatic.py b/imod/protocols/protocol_ctfEstimation_automatic.py index 884aca05..14fbefcf 100644 --- a/imod/protocols/protocol_ctfEstimation_automatic.py +++ b/imod/protocols/protocol_ctfEstimation_automatic.py @@ -499,6 +499,7 @@ def getExpectedDefocus(self): if self.expectedDefocusOrigin.get() == 1: with open(self.expectedDefocusFile.get()) as f: lines = f.readlines() + lines = filter(lambda x: x.strip(), lines) result = {line.split()[0]: line.split()[1] for line in lines} return result else: From 3b97772bd78465031945a8588d686cf0a973ed41 Mon Sep 17 00:00:00 2001 From: Grigory Sharov Date: Sat, 8 Jun 2024 17:29:40 +0100 Subject: [PATCH 47/49] fix summary --- imod/protocols/protocol_etomo.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/imod/protocols/protocol_etomo.py b/imod/protocols/protocol_etomo.py index bbc434d1..cd9dd90d 100644 --- a/imod/protocols/protocol_etomo.py +++ b/imod/protocols/protocol_etomo.py @@ -67,7 +67,7 @@ def __init__(self, **kwargs): super().__init__(**kwargs) self.PrealignedTiltSeries = None self.FullTomograms = None - self.PostProcessedTomograms = None + self.PostProcessTomograms = None # -------------------------- DEFINE param functions ----------------------- def _defineParams(self, form): @@ -645,7 +645,7 @@ def _summary(self): if self.FullTomograms: summary.append("- Raw reconstructed tomogram") - if self.PostProcessedTomograms: + if self.PostProcessTomograms: summary.append("- Post-processed tomogram") if len(summary) == 1: From e55d15b5d7138ced5d4f0dcb165eb0230cf1f43e Mon Sep 17 00:00:00 2001 From: Grigory Sharov Date: Fri, 14 Jun 2024 11:07:29 +0100 Subject: [PATCH 48/49] housekeeping --- .github/workflows/python-package.yml | 48 ++++------------------------ MANIFEST.in | 2 +- README.rst | 6 ---- setup.py | 2 +- 4 files changed, 9 insertions(+), 49 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index bc6fdd8d..a6676261 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -1,48 +1,14 @@ -# Workflow to send master to pypi and tag the branch: -# You need to edit FOLDER_WITH_VERSION with the folder that has the __version__ value. - +# Workflow to send master to pypi and tag the branch name: master to pypi with comments and tag - -# Controls when the action will run. Triggers the workflow on push or pull request -# events but only for the master branch +# Triggers the workflow on push to the master branch on: push: branches: [ master ] -env: - FOLDER_WITH_VERSION: imod -# A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: - deploy: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - name: Set up MPI - uses: mpi4py/setup-mpi@v1 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.8' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install setuptools wheel twine - pip install scipion-pyworkflow - pip install scipion-em - - name: Build and publish - env: - TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - run: | - python setup.py sdist - twine upload dist/* -c "${{ secrets.PYPI_COMMENT }}" - - name: Get version and tag - env: - FOLDER_WITH_VERSION: imod - run: | - export PACKAGE_VERSION=$(python -c "import $FOLDER_WITH_VERSION; print('VERSION', 'v'+$FOLDER_WITH_VERSION.__version__)" | grep VERSION | sed "s/VERSION //g") - git tag $PACKAGE_VERSION - git push origin $PACKAGE_VERSION + call-publish-workflow: + uses: scipion-em/.github/.github/workflows/publish_and_tag.yml@master + with: + folder: imod + secrets: inherit diff --git a/MANIFEST.in b/MANIFEST.in index c3d3b670..607c4755 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,7 +4,7 @@ include *.rst # Include the license file include LICENSE -# Include the changelog +# Include the changelog and requirements include *.txt # Include conf file diff --git a/README.rst b/README.rst index 8110fd97..901ced79 100644 --- a/README.rst +++ b/README.rst @@ -24,12 +24,6 @@ This plugin provides wrappers for several programs of `IMOD Date: Mon, 17 Jun 2024 13:29:11 +0200 Subject: [PATCH 49/49] removing non used parameters --- imod/protocols/protocol_tomoPreprocess.py | 7 ++++--- imod/protocols/protocol_tsPreprocess.py | 19 ++++++++++--------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/imod/protocols/protocol_tomoPreprocess.py b/imod/protocols/protocol_tomoPreprocess.py index 9946a9b5..737a4451 100644 --- a/imod/protocols/protocol_tomoPreprocess.py +++ b/imod/protocols/protocol_tomoPreprocess.py @@ -178,7 +178,7 @@ def _defineParams(self, form): form.addParam('scaleRangeToggle', params.EnumParam, choices=['Yes', 'No'], - condition="floatDensities==0 or floatDensities==1 or floatDensities==3", + condition="floatDensities==1 or floatDensities==3", default=0, label='Set scaling range values?', display=params.EnumParam.DISPLAY_HLIST, @@ -189,14 +189,14 @@ def _defineParams(self, form): form.addParam('scaleRangeMax', params.FloatParam, - condition="(floatDensities==0 or floatDensities==1 or floatDensities==3) and scaleRangeToggle==0", + condition="(floatDensities==1 or floatDensities==3) and scaleRangeToggle==0", default=255, label='Max.', help='Maximum value for the rescaling') form.addParam('scaleRangeMin', params.FloatParam, - condition="(floatDensities==0 or floatDensities==1 or floatDensities==3) and scaleRangeToggle==0", + condition="(floatDensities==1 or floatDensities==3) and scaleRangeToggle==0", default=0, label='Min.', help='Minimum value for the rescaling') @@ -207,6 +207,7 @@ def _defineParams(self, form): 'Lanczos 2 lobes', 'Lanczos 3 lobes'], default=5, label='Antialias method:', + expertLevel=params.LEVEL_ADVANCED, display=params.EnumParam.DISPLAY_COMBO, help='Type of antialiasing filter to use when reducing images.\n' 'The available types of filters are:\n\n' diff --git a/imod/protocols/protocol_tsPreprocess.py b/imod/protocols/protocol_tsPreprocess.py index 5b916c79..55c8a325 100644 --- a/imod/protocols/protocol_tsPreprocess.py +++ b/imod/protocols/protocol_tsPreprocess.py @@ -179,8 +179,9 @@ def _defineParams(self, form): form.addParam('scaleRangeToggle', params.EnumParam, + choices=['Yes', 'No'], - condition="floatDensities==0 or floatDensities==1 or floatDensities==3", + condition="floatDensities==1 or floatDensities==3", default=0, label='Set scaling range values?', display=params.EnumParam.DISPLAY_HLIST, @@ -189,20 +190,20 @@ def _defineParams(self, form): 'minimum and maximum density will be mapped ' 'to the Min and Max values that are entered') - form.addParam('scaleRangeMax', - params.FloatParam, - condition="(floatDensities==0 or floatDensities==1 or floatDensities==3) and scaleRangeToggle==0", - default=255, - label='Max.', - help='Maximum value for the rescaling') - form.addParam('scaleRangeMin', params.FloatParam, - condition="(floatDensities==0 or floatDensities==1 or floatDensities==3) and scaleRangeToggle==0", + condition="(floatDensities==1 or floatDensities==3) and scaleRangeToggle==0", default=0, label='Min.', help='Minimum value for the rescaling') + form.addParam('scaleRangeMax', + params.FloatParam, + condition="(floatDensities==1 or floatDensities==3) and scaleRangeToggle==0", + default=255, + label='Max.', + help='Maximum value for the rescaling') + form.addParam('antialias', params.EnumParam, expertLevel=params.LEVEL_ADVANCED,