From 0b6e510ef74682564e5c3dec3d900a86c3b6b25a Mon Sep 17 00:00:00 2001 From: Russell Grew Date: Wed, 14 Apr 2021 14:53:28 +1000 Subject: [PATCH 1/6] wait for background tasks to finish --- contrib/stack/topsStack/Stack.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/contrib/stack/topsStack/Stack.py b/contrib/stack/topsStack/Stack.py index ccd347c1..d8a0e3a4 100644 --- a/contrib/stack/topsStack/Stack.py +++ b/contrib/stack/topsStack/Stack.py @@ -260,7 +260,7 @@ def denseOffset(self, function): #self.f.write('ww : 256\n') #self.f.write('wh : 128\n') - def write_wrapper_config2run_file(self, configName, line_cnt, numProcess = 1): + def write_wrapper_config2run_file(self, configName, line_cnt, last_line = False, numProcess = 1): # dispassionate list of commands for single process if numProcess == 1: self.runf.write(self.text_cmd + 'SentinelWrapper.py -c ' + configName + '\n') @@ -270,6 +270,9 @@ def write_wrapper_config2run_file(self, configName, line_cnt, numProcess = 1): if line_cnt == numProcess: self.runf.write('wait\n\n') line_cnt = 0 + # last line of file, wait for any background processes to complete + if last_line: + self.runf.write('wait\n') return line_cnt def finalize(self): @@ -358,7 +361,7 @@ def unpackSecondarysSLC(self, stackReferenceDate, secondaryList, safe_dict): configObj.finalize() line_cnt += 1 - line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt, self.numProcess) + line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt, secondary == secondaryList[-1], self.numProcess) del configObj def averageBaseline(self, stackReferenceDate, secondaryList): @@ -375,7 +378,7 @@ def averageBaseline(self, stackReferenceDate, secondaryList): configObj.finalize() line_cnt += 1 - line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt, self.numProcess) + line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt, secondary == secondaryList[-1], self.numProcess) del configObj def gridBaseline(self, stackReferenceDate, secondaryList): @@ -440,7 +443,7 @@ def geo2rdr_offset(self, secondaryList, fullBurst='False'): configObj.finalize() line_cnt += 1 - line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt, self.numProcess) + line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt, secondary == secondaryList[-1], self.numProcess) del configObj def resample_with_carrier(self, secondaryList, fullBurst='False'): @@ -470,7 +473,7 @@ def resample_with_carrier(self, secondaryList, fullBurst='False'): configObj.finalize() line_cnt += 1 - line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt, self.numProcess) + line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt, secondary == secondaryList[-1], self.numProcess) del configObj @@ -521,7 +524,7 @@ def pairs_misregistration(self, dateList, safe_dict): configObj.finalize() line_cnt += 1 - line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt, self.numProcess) + line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt, pair == pairs[-1], self.numProcess) del configObj ######################## @@ -560,7 +563,7 @@ def generate_burstIgram(self, dateList, safe_dict, pairs): configObj.finalize() line_cnt += 1 - line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt, self.numProcess) + line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt, pair == pairs[-1], self.numProcess) del configObj @@ -594,7 +597,7 @@ def igram_mergeBurst(self, dateList, safe_dict, pairs): configObj.finalize() line_cnt += 1 - line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt, self.numProcess) + line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt, pair == pairs[-1], self.numProcess) del configObj def mergeSecondarySLC(self, secondaryList, virtual='True'): @@ -746,7 +749,7 @@ def filter_coherence(self, pairs): configObj.finalize() line_cnt += 1 - line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt, self.numProcess) + line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt, pair == pairs[-1], self.numProcess) del configObj @@ -772,7 +775,7 @@ def unwrap(self, pairs): configObj.finalize() line_cnt += 1 - line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt, self.numProcess) + line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt, pair == pairs[-1], self.numProcess) del configObj def denseOffsets(self, pairs): From d4b1f99ef0f5cc97e7be93248517f69e2606d4b9 Mon Sep 17 00:00:00 2001 From: Russell Grew Date: Fri, 22 Jul 2022 13:15:13 +1000 Subject: [PATCH 2/6] tidy based on PR feedback --- contrib/stack/topsStack/Stack.py | 43 ++++++++++++++++---------------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/contrib/stack/topsStack/Stack.py b/contrib/stack/topsStack/Stack.py index d8a0e3a4..441151bd 100644 --- a/contrib/stack/topsStack/Stack.py +++ b/contrib/stack/topsStack/Stack.py @@ -260,20 +260,19 @@ def denseOffset(self, function): #self.f.write('ww : 256\n') #self.f.write('wh : 128\n') - def write_wrapper_config2run_file(self, configName, line_cnt, last_line = False, numProcess = 1): + def write_wrapper_config2run_file(self, configName, line_cnt = 1, last_line = False, numProcess = 1): # dispassionate list of commands for single process if numProcess == 1: self.runf.write(self.text_cmd + 'SentinelWrapper.py -c ' + configName + '\n') # aggregate background commands between wait blocks for speed gains elif numProcess > 1: self.runf.write(self.text_cmd + 'SentinelWrapper.py -c ' + configName + ' &\n') - if line_cnt == numProcess: + if line_cnt % numProcess == 0: self.runf.write('wait\n\n') line_cnt = 0 # last line of file, wait for any background processes to complete if last_line: self.runf.write('wait\n') - return line_cnt def finalize(self): self.f.close() @@ -320,7 +319,7 @@ def unpackSLC(self, acquisitionDates, safe_dict): configObj.finalize() line_cnt += 1 - line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt) + configObj.write_wrapper_config2run_file(configName, line_cnt) del configObj def unpackStackReferenceSLC(self, safe_dict): @@ -341,7 +340,7 @@ def unpackStackReferenceSLC(self, safe_dict): configObj.finalize() line_cnt = 1 - line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt) + configObj.write_wrapper_config2run_file(configName, line_cnt) del configObj def unpackSecondarysSLC(self, stackReferenceDate, secondaryList, safe_dict): @@ -361,7 +360,7 @@ def unpackSecondarysSLC(self, stackReferenceDate, secondaryList, safe_dict): configObj.finalize() line_cnt += 1 - line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt, secondary == secondaryList[-1], self.numProcess) + configObj.write_wrapper_config2run_file(configName, line_cnt, secondary == secondaryList[-1], self.numProcess) del configObj def averageBaseline(self, stackReferenceDate, secondaryList): @@ -378,7 +377,7 @@ def averageBaseline(self, stackReferenceDate, secondaryList): configObj.finalize() line_cnt += 1 - line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt, secondary == secondaryList[-1], self.numProcess) + configObj.write_wrapper_config2run_file(configName, line_cnt, secondary == secondaryList[-1], self.numProcess) del configObj def gridBaseline(self, stackReferenceDate, secondaryList): @@ -395,7 +394,7 @@ def gridBaseline(self, stackReferenceDate, secondaryList): configObj.finalize() line_cnt += 1 - line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt) + configObj.write_wrapper_config2run_file(configName, line_cnt) del configObj # also add the reference in itself to be consistent with the SLC dir configName = os.path.join(self.config_path,'config_baselinegrid_reference') @@ -408,7 +407,7 @@ def gridBaseline(self, stackReferenceDate, secondaryList): configObj.finalize() line_cnt = 1 - line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt) + configObj.write_wrapper_config2run_file(configName, line_cnt) del configObj @@ -443,7 +442,7 @@ def geo2rdr_offset(self, secondaryList, fullBurst='False'): configObj.finalize() line_cnt += 1 - line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt, secondary == secondaryList[-1], self.numProcess) + configObj.write_wrapper_config2run_file(configName, line_cnt, secondary == secondaryList[-1], self.numProcess) del configObj def resample_with_carrier(self, secondaryList, fullBurst='False'): @@ -473,7 +472,7 @@ def resample_with_carrier(self, secondaryList, fullBurst='False'): configObj.finalize() line_cnt += 1 - line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt, secondary == secondaryList[-1], self.numProcess) + configObj.write_wrapper_config2run_file(configName, line_cnt, secondary == secondaryList[-1], self.numProcess) del configObj @@ -524,7 +523,7 @@ def pairs_misregistration(self, dateList, safe_dict): configObj.finalize() line_cnt += 1 - line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt, pair == pairs[-1], self.numProcess) + configObj.write_wrapper_config2run_file(configName, line_cnt, pair == pairs[-1], self.numProcess) del configObj ######################## @@ -563,7 +562,7 @@ def generate_burstIgram(self, dateList, safe_dict, pairs): configObj.finalize() line_cnt += 1 - line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt, pair == pairs[-1], self.numProcess) + configObj.write_wrapper_config2run_file(configName, line_cnt, pair == pairs[-1], self.numProcess) del configObj @@ -597,7 +596,7 @@ def igram_mergeBurst(self, dateList, safe_dict, pairs): configObj.finalize() line_cnt += 1 - line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt, pair == pairs[-1], self.numProcess) + configObj.write_wrapper_config2run_file(configName, line_cnt, pair == pairs[-1], self.numProcess) del configObj def mergeSecondarySLC(self, secondaryList, virtual='True'): @@ -622,7 +621,7 @@ def mergeSecondarySLC(self, secondaryList, virtual='True'): configObj.finalize() line_cnt += 1 - line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt) + configObj.write_wrapper_config2run_file(configName, line_cnt) del configObj @@ -645,7 +644,7 @@ def mergeReference(self, stackReference, virtual='True'): configObj.finalize() line_cnt = 1 - line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt) + configObj.write_wrapper_config2run_file(configName, line_cnt) del configObj geometryList = ['lat*rdr', 'lon*rdr', 'los*rdr', 'hgt*rdr', 'shadowMask*rdr','incLocal*rdr'] @@ -673,7 +672,7 @@ def mergeReference(self, stackReference, virtual='True'): configObj.finalize() line_cnt += 1 - line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt) + configObj.write_wrapper_config2run_file(configName, line_cnt) del configObj def mergeSLC(self, aquisitionDates, virtual='True'): @@ -697,7 +696,7 @@ def mergeSLC(self, aquisitionDates, virtual='True'): configObj.finalize() line_cnt += 1 - line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt) + configObj.write_wrapper_config2run_file(configName, line_cnt) geometryList = ['lat*rdr', 'lon*rdr', 'los*rdr', 'hgt*rdr', 'shadowMask*rdr','incLocal*rdr'] @@ -722,7 +721,7 @@ def mergeSLC(self, aquisitionDates, virtual='True'): configObj.finalize() g_line_cnt += 1 - g_line_cnt = configObj.write_wrapper_config2run_file(configName, g_line_cnt) + configObj.write_wrapper_config2run_file(configName, g_line_cnt) del configObj def filter_coherence(self, pairs): @@ -749,7 +748,7 @@ def filter_coherence(self, pairs): configObj.finalize() line_cnt += 1 - line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt, pair == pairs[-1], self.numProcess) + configObj.write_wrapper_config2run_file(configName, line_cnt, pair == pairs[-1], self.numProcess) del configObj @@ -775,7 +774,7 @@ def unwrap(self, pairs): configObj.finalize() line_cnt += 1 - line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt, pair == pairs[-1], self.numProcess) + configObj.write_wrapper_config2run_file(configName, line_cnt, pair == pairs[-1], self.numProcess) del configObj def denseOffsets(self, pairs): @@ -795,7 +794,7 @@ def denseOffsets(self, pairs): configObj.finalize() line_cnt += 1 - line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt) + configObj.write_wrapper_config2run_file(configName, line_cnt) del configObj def finalize(self): From f8cd5d8d825f3dd3208551b1b8fda632db0debf6 Mon Sep 17 00:00:00 2001 From: Russell Grew Date: Fri, 22 Jul 2022 13:38:55 +1000 Subject: [PATCH 3/6] last_line logic in numProcess wrapper --- contrib/stack/topsStack/Stack.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/contrib/stack/topsStack/Stack.py b/contrib/stack/topsStack/Stack.py index 441151bd..06188827 100644 --- a/contrib/stack/topsStack/Stack.py +++ b/contrib/stack/topsStack/Stack.py @@ -267,12 +267,8 @@ def write_wrapper_config2run_file(self, configName, line_cnt = 1, last_line = Fa # aggregate background commands between wait blocks for speed gains elif numProcess > 1: self.runf.write(self.text_cmd + 'SentinelWrapper.py -c ' + configName + ' &\n') - if line_cnt % numProcess == 0: + if line_cnt % numProcess == 0 or last_line: self.runf.write('wait\n\n') - line_cnt = 0 - # last line of file, wait for any background processes to complete - if last_line: - self.runf.write('wait\n') def finalize(self): self.f.close() From 4ffbffff0b269d13e53775394cb65da4278a1b6f Mon Sep 17 00:00:00 2001 From: Russell Grew Date: Wed, 27 Jul 2022 13:45:18 +1000 Subject: [PATCH 4/6] tidy config wrapper calls --- contrib/stack/topsStack/Stack.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/contrib/stack/topsStack/Stack.py b/contrib/stack/topsStack/Stack.py index 06188827..a409acd0 100644 --- a/contrib/stack/topsStack/Stack.py +++ b/contrib/stack/topsStack/Stack.py @@ -335,8 +335,7 @@ def unpackStackReferenceSLC(self, safe_dict): configObj.topo('[Function-2]') configObj.finalize() - line_cnt = 1 - configObj.write_wrapper_config2run_file(configName, line_cnt) + configObj.write_wrapper_config2run_file(configName) del configObj def unpackSecondarysSLC(self, stackReferenceDate, secondaryList, safe_dict): @@ -356,7 +355,8 @@ def unpackSecondarysSLC(self, stackReferenceDate, secondaryList, safe_dict): configObj.finalize() line_cnt += 1 - configObj.write_wrapper_config2run_file(configName, line_cnt, secondary == secondaryList[-1], self.numProcess) + last_line = secondary == secondaryList[-1] + configObj.write_wrapper_config2run_file(configName, line_cnt, last_line = last_line, numProcess = self.numProcess) del configObj def averageBaseline(self, stackReferenceDate, secondaryList): @@ -373,7 +373,8 @@ def averageBaseline(self, stackReferenceDate, secondaryList): configObj.finalize() line_cnt += 1 - configObj.write_wrapper_config2run_file(configName, line_cnt, secondary == secondaryList[-1], self.numProcess) + last_line = secondary == secondaryList[-1] + configObj.write_wrapper_config2run_file(configName, line_cnt, last_line = last_line, numProcess = self.numProcess) del configObj def gridBaseline(self, stackReferenceDate, secondaryList): @@ -402,8 +403,7 @@ def gridBaseline(self, stackReferenceDate, secondaryList): configObj.computeGridBaseline('[Function-1]') configObj.finalize() - line_cnt = 1 - configObj.write_wrapper_config2run_file(configName, line_cnt) + configObj.write_wrapper_config2run_file(configName) del configObj @@ -438,7 +438,8 @@ def geo2rdr_offset(self, secondaryList, fullBurst='False'): configObj.finalize() line_cnt += 1 - configObj.write_wrapper_config2run_file(configName, line_cnt, secondary == secondaryList[-1], self.numProcess) + last_line = secondary == secondaryList[-1] + configObj.write_wrapper_config2run_file(configName, line_cnt, last_line = last_line, numProcess = self.numProcess) del configObj def resample_with_carrier(self, secondaryList, fullBurst='False'): @@ -468,7 +469,8 @@ def resample_with_carrier(self, secondaryList, fullBurst='False'): configObj.finalize() line_cnt += 1 - configObj.write_wrapper_config2run_file(configName, line_cnt, secondary == secondaryList[-1], self.numProcess) + last_line = secondary == secondaryList[-1] + configObj.write_wrapper_config2run_file(configName, line_cnt, last_line = last_line, numProcess = self.numProcess) del configObj @@ -639,8 +641,8 @@ def mergeReference(self, stackReference, virtual='True'): configObj.mergeBurst('[Function-1]') configObj.finalize() - line_cnt = 1 - configObj.write_wrapper_config2run_file(configName, line_cnt) + line_cnt = 1; + configObj.write_wrapper_config2run_file(configName) del configObj geometryList = ['lat*rdr', 'lon*rdr', 'los*rdr', 'hgt*rdr', 'shadowMask*rdr','incLocal*rdr'] From 8971f5292f5803b036ac3b14e5a3893d842a2bac Mon Sep 17 00:00:00 2001 From: Russell Grew Date: Wed, 27 Jul 2022 14:21:21 +1000 Subject: [PATCH 5/6] remove messy line_cnt logic and enumerate loops --- contrib/stack/topsStack/Stack.py | 97 ++++++++++++-------------------- 1 file changed, 35 insertions(+), 62 deletions(-) diff --git a/contrib/stack/topsStack/Stack.py b/contrib/stack/topsStack/Stack.py index a409acd0..845412af 100644 --- a/contrib/stack/topsStack/Stack.py +++ b/contrib/stack/topsStack/Stack.py @@ -298,8 +298,7 @@ def unpackSLC(self, acquisitionDates, safe_dict): swath_path = self.work_dir os.makedirs(self.config_path, exist_ok=True) - line_cnt = 0 - for slcdate in acquisitionDates: + for i, slcdate in enumerate(acquisitionDates): configName = os.path.join(self.config_path,'config_unpack_'+slcdate) configObj = config(configName) configObj.configure(self) @@ -314,8 +313,7 @@ def unpackSLC(self, acquisitionDates, safe_dict): configObj.topo('[Function-2]') configObj.finalize() - line_cnt += 1 - configObj.write_wrapper_config2run_file(configName, line_cnt) + configObj.write_wrapper_config2run_file(configName, line_cnt = i + 1) del configObj def unpackStackReferenceSLC(self, safe_dict): @@ -340,8 +338,7 @@ def unpackStackReferenceSLC(self, safe_dict): def unpackSecondarysSLC(self, stackReferenceDate, secondaryList, safe_dict): - line_cnt = 0 - for secondary in secondaryList: + for i, secondary in enumerate(secondaryList): configName = os.path.join(self.config_path,'config_secondary_'+secondary) outdir = os.path.join(self.work_dir,'secondarys/'+secondary) configObj = config(configName) @@ -354,15 +351,13 @@ def unpackSecondarysSLC(self, stackReferenceDate, secondaryList, safe_dict): configObj.Sentinel1_TOPS('[Function-1]') configObj.finalize() - line_cnt += 1 last_line = secondary == secondaryList[-1] - configObj.write_wrapper_config2run_file(configName, line_cnt, last_line = last_line, numProcess = self.numProcess) + configObj.write_wrapper_config2run_file(configName, line_cnt = i + 1, last_line = last_line, numProcess = self.numProcess) del configObj def averageBaseline(self, stackReferenceDate, secondaryList): - line_cnt = 0 - for secondary in secondaryList: + for i, secondary in enumerate(secondaryList): configName = os.path.join(self.config_path,'config_baseline_'+secondary) configObj = config(configName) configObj.configure(self) @@ -372,15 +367,13 @@ def averageBaseline(self, stackReferenceDate, secondaryList): configObj.computeAverageBaseline('[Function-1]') configObj.finalize() - line_cnt += 1 last_line = secondary == secondaryList[-1] - configObj.write_wrapper_config2run_file(configName, line_cnt, last_line = last_line, numProcess = self.numProcess) + configObj.write_wrapper_config2run_file(configName, line_cnt = i + 1, last_line = last_line, numProcess = self.numProcess) del configObj def gridBaseline(self, stackReferenceDate, secondaryList): - line_cnt = 0 - for secondary in secondaryList: + for i, secondary in enumerate(secondaryList): configName = os.path.join(self.config_path,'config_baselinegrid_'+secondary) configObj = config(configName) configObj.configure(self) @@ -390,8 +383,7 @@ def gridBaseline(self, stackReferenceDate, secondaryList): configObj.computeGridBaseline('[Function-1]') configObj.finalize() - line_cnt += 1 - configObj.write_wrapper_config2run_file(configName, line_cnt) + configObj.write_wrapper_config2run_file(configName, line_cnt = i + 1) del configObj # also add the reference in itself to be consistent with the SLC dir configName = os.path.join(self.config_path,'config_baselinegrid_reference') @@ -414,8 +406,7 @@ def extractOverlaps(self): def geo2rdr_offset(self, secondaryList, fullBurst='False'): - line_cnt = 0 - for secondary in secondaryList: + for i, secondary in enumerate(secondaryList): reference = self.reference_date if fullBurst == 'True': configName = os.path.join(self.config_path, 'config_fullBurst_geo2rdr_' + secondary) @@ -437,15 +428,13 @@ def geo2rdr_offset(self, secondaryList, fullBurst='False'): configObj.geo2rdr('[Function-1]') configObj.finalize() - line_cnt += 1 last_line = secondary == secondaryList[-1] - configObj.write_wrapper_config2run_file(configName, line_cnt, last_line = last_line, numProcess = self.numProcess) + configObj.write_wrapper_config2run_file(configName, line_cnt = i + 1, last_line = last_line, numProcess = self.numProcess) del configObj def resample_with_carrier(self, secondaryList, fullBurst='False'): - line_cnt = 0 - for secondary in secondaryList: + for i, secondary in enumerate(secondaryList): reference = self.reference_date if fullBurst == 'True': configName = os.path.join(self.config_path, 'config_fullBurst_resample_' + secondary) @@ -468,9 +457,8 @@ def resample_with_carrier(self, secondaryList, fullBurst='False'): configObj.resamp_withCarrier('[Function-1]') configObj.finalize() - line_cnt += 1 last_line = secondary == secondaryList[-1] - configObj.write_wrapper_config2run_file(configName, line_cnt, last_line = last_line, numProcess = self.numProcess) + configObj.write_wrapper_config2run_file(configName, line_cnt = i + 1, last_line = last_line, numProcess = self.numProcess) del configObj @@ -490,8 +478,7 @@ def pairs_misregistration(self, dateList, safe_dict): safe_dict[self.reference_date].slc = os.path.join(self.work_dir , 'reference') safe_dict[self.reference_date].slc_overlap = os.path.join(self.work_dir , 'reference') - line_cnt = 0 - for pair in pairs: + for i, pair in enumerate(pairs): reference = pair[0] secondary = pair[1] interferogramDir = os.path.join(self.work_dir, 'coarse_interferograms/'+reference+'_'+secondary) @@ -520,8 +507,8 @@ def pairs_misregistration(self, dateList, safe_dict): configObj.rangeMisreg('[Function-4]') configObj.finalize() - line_cnt += 1 - configObj.write_wrapper_config2run_file(configName, line_cnt, pair == pairs[-1], self.numProcess) + last_line = pair == pairs[-1] + configObj.write_wrapper_config2run_file(configName, line_cnt = i + 1, last_line = last_line, numProcess = self.numProcess) del configObj ######################## @@ -542,8 +529,7 @@ def generate_burstIgram(self, dateList, safe_dict, pairs): safe_dict[date].slc = os.path.join(self.work_dir, 'coreg_secondarys/'+date) safe_dict[self.reference_date].slc = os.path.join(self.work_dir , 'reference') - line_cnt = 0 - for pair in pairs: + for i, pair in enumerate(pairs): reference = pair[0] secondary = pair[1] interferogramDir = os.path.join(self.work_dir, 'interferograms/' + reference + '_' + secondary) @@ -559,8 +545,8 @@ def generate_burstIgram(self, dateList, safe_dict, pairs): configObj.generateIgram('[Function-1]') configObj.finalize() - line_cnt += 1 - configObj.write_wrapper_config2run_file(configName, line_cnt, pair == pairs[-1], self.numProcess) + last_line = pair == pairs[-1] + configObj.write_wrapper_config2run_file(configName, line_cnt = i + 1, last_line = last_line, numProcess = self.numProcess) del configObj @@ -570,8 +556,7 @@ def igram_mergeBurst(self, dateList, safe_dict, pairs): safe_dict[date].slc = os.path.join(self.work_dir, 'coreg_secondarys/'+date) safe_dict[self.reference_date].slc = os.path.join(self.work_dir , 'reference') - line_cnt = 0 - for pair in pairs: + for i, pair in enumerate(pairs): reference = pair[0] secondary = pair[1] interferogramDir = os.path.join(self.work_dir, 'interferograms/' + reference + '_' + secondary) @@ -593,14 +578,13 @@ def igram_mergeBurst(self, dateList, safe_dict, pairs): configObj.mergeBurst('[Function-1]') configObj.finalize() - line_cnt += 1 - configObj.write_wrapper_config2run_file(configName, line_cnt, pair == pairs[-1], self.numProcess) + last_line = pair == pairs[-1] + configObj.write_wrapper_config2run_file(configName, line_cnt = i + 1, last_line = last_line, numProcess = self.numProcess) del configObj def mergeSecondarySLC(self, secondaryList, virtual='True'): - line_cnt = 0 - for secondary in secondaryList: + for i, secondary in enumerate(secondaryList): configName = os.path.join(self.config_path,'config_merge_' + secondary) configObj = config(configName) configObj.configure(self) @@ -618,8 +602,7 @@ def mergeSecondarySLC(self, secondaryList, virtual='True'): configObj.mergeBurst('[Function-1]') configObj.finalize() - line_cnt += 1 - configObj.write_wrapper_config2run_file(configName, line_cnt) + configObj.write_wrapper_config2run_file(configName, line_cnt = i + 1) del configObj @@ -641,7 +624,6 @@ def mergeReference(self, stackReference, virtual='True'): configObj.mergeBurst('[Function-1]') configObj.finalize() - line_cnt = 1; configObj.write_wrapper_config2run_file(configName) del configObj @@ -669,14 +651,12 @@ def mergeReference(self, stackReference, virtual='True'): configObj.mergeBurst('[Function-1]') configObj.finalize() - line_cnt += 1 - configObj.write_wrapper_config2run_file(configName, line_cnt) + configObj.write_wrapper_config2run_file(configName, line_cnt = i + 1) del configObj def mergeSLC(self, aquisitionDates, virtual='True'): - line_cnt = 0 - for slcdate in aquisitionDates: + for i, slcdate in enumerate(aquisitionDates): configName = os.path.join(self.config_path,'config_merge_' + slcdate) configObj = config(configName) configObj.configure(self) @@ -693,13 +673,11 @@ def mergeSLC(self, aquisitionDates, virtual='True'): configObj.mergeBurst('[Function-1]') configObj.finalize() - line_cnt += 1 - configObj.write_wrapper_config2run_file(configName, line_cnt) + configObj.write_wrapper_config2run_file(configName, line_cnt = i + 1) geometryList = ['lat*rdr', 'lon*rdr', 'los*rdr', 'hgt*rdr', 'shadowMask*rdr','incLocal*rdr'] - g_line_cnt = 0 for i in range(len(geometryList)): pattern = geometryList[i] configName = os.path.join(self.config_path,'config_merge_' + slcdate + '_' +pattern.split('*')[0]) @@ -718,14 +696,12 @@ def mergeSLC(self, aquisitionDates, virtual='True'): configObj.mergeBurst('[Function-1]') configObj.finalize() - g_line_cnt += 1 - configObj.write_wrapper_config2run_file(configName, g_line_cnt) + configObj.write_wrapper_config2run_file(configName, i + 1) del configObj def filter_coherence(self, pairs): - line_cnt = 0 - for pair in pairs: + for i, pair in enumerate(pairs): reference = pair[0] secondary = pair[1] mergedDir = os.path.join(self.work_dir, 'merged/interferograms/' + reference + '_' + secondary) @@ -745,15 +721,14 @@ def filter_coherence(self, pairs): configObj.FilterAndCoherence('[Function-1]') configObj.finalize() - line_cnt += 1 - configObj.write_wrapper_config2run_file(configName, line_cnt, pair == pairs[-1], self.numProcess) + last_line = pair == pairs[-1] + configObj.write_wrapper_config2run_file(configName, line_cnt = i + 1, last_line = last_line, numProcess = self.numProcess) del configObj def unwrap(self, pairs): - line_cnt = 0 - for pair in pairs: + for i, pair in enumerate(pairs): reference = pair[0] secondary = pair[1] mergedDir = os.path.join(self.work_dir, 'merged/interferograms/' + reference + '_' + secondary) @@ -771,14 +746,13 @@ def unwrap(self, pairs): configObj.unwrap('[Function-1]') configObj.finalize() - line_cnt += 1 - configObj.write_wrapper_config2run_file(configName, line_cnt, pair == pairs[-1], self.numProcess) + last_line = pair == pairs[-1] + configObj.write_wrapper_config2run_file(configName, line_cnt = i + 1, last_line = last_line, numProcess = self.numProcess) del configObj def denseOffsets(self, pairs): - line_cnt = 0 - for pair in pairs: + for i, pair in enumerate(pairs): reference = pair[0] secondary = pair[1] configName = os.path.join(self.config_path ,'config_denseOffset_' + reference + '_' + secondary) @@ -791,8 +765,7 @@ def denseOffsets(self, pairs): configObj.denseOffset('[Function-1]') configObj.finalize() - line_cnt += 1 - configObj.write_wrapper_config2run_file(configName, line_cnt) + configObj.write_wrapper_config2run_file(configName, line_cnt = i + 1) del configObj def finalize(self): From 426d033efe6d36c3f6268372fad47cbfd60d34e3 Mon Sep 17 00:00:00 2001 From: Russell Grew Date: Wed, 27 Jul 2022 16:11:34 +1000 Subject: [PATCH 6/6] consider ionospheric phase estimation --- contrib/stack/topsStack/Stack.py | 42 ++++++++++++-------------------- 1 file changed, 15 insertions(+), 27 deletions(-) diff --git a/contrib/stack/topsStack/Stack.py b/contrib/stack/topsStack/Stack.py index 84d1b528..e524d3cc 100644 --- a/contrib/stack/topsStack/Stack.py +++ b/contrib/stack/topsStack/Stack.py @@ -340,9 +340,7 @@ def filtIon(self, function): self.f.write('win_max : ' + '{}'.format(self.win_max) + '\n') - - - def write_wrapper_config2run_file(self, configName, line_cnt, numProcess = 1): + def write_wrapper_config2run_file(self, configName, line_cnt = 1, last_line = False, numProcess = 1): # dispassionate list of commands for single process if numProcess == 1: self.runf.write(self.text_cmd + 'SentinelWrapper.py -c ' + configName + '\n') @@ -1134,8 +1132,7 @@ def denseOffsets(self, pairs): def subband_and_resamp(self, dateListIon, stackReferenceDate): - line_cnt = 0 - for date in dateListIon: + for i, date in enumerate(dateListIon): configName = os.path.join(self.config_path,'config_subband_and_resamp_{}'.format(date)) configObj = config(configName) configObj.configure(self) @@ -1150,15 +1147,14 @@ def subband_and_resamp(self, dateListIon, stackReferenceDate): configObj.subband_and_resamp('[Function-1]') configObj.finalize() - line_cnt += 1 - line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt, self.numProcess) + last_line = date == dateListIon[-1] + configObj.write_wrapper_config2run_file(configName, line_cnt = i + 1, last_line = last_line, numProcess = self.numProcess) del configObj def generateIgram_ion(self, pairs, stackReferenceDate): - line_cnt = 0 - for p in pairs: + for i, p in enumerate(pairs): configName = os.path.join(self.config_path,'config_generateIgram_ion_{}_{}'.format(p[0], p[1])) configObj = config(configName) configObj.configure(self) @@ -1185,8 +1181,8 @@ def generateIgram_ion(self, pairs, stackReferenceDate): configObj.generateIgram_ion('[Function-2]') configObj.finalize() - line_cnt += 1 - line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt, self.numProcess) + last_line = p == pairs[-1] + configObj.write_wrapper_config2run_file(configName, line_cnt = i + 1, last_line = last_line, numProcess = self.numProcess) del configObj @@ -1206,8 +1202,7 @@ def mergeBurstsIon(self, pairs_same_starting_ranges_update, pairs_diff_starting_ pairs1 = pairs_same_starting_ranges_update pairs2 = pairs_diff_starting_ranges_update - line_cnt = 0 - for p in pairs1+pairs2: + for i, p in enumerate(pairs1+pairs2): configName = os.path.join(self.config_path,'config_mergeBurstsIon_{}-{}'.format(p[0], p[1])) configObj = config(configName) configObj.configure(self) @@ -1265,9 +1260,8 @@ def mergeBurstsIon(self, pairs_same_starting_ranges_update, pairs_diff_starting_ configObj.finalize() - line_cnt += 1 #line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt, self.numProcess) - line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt) + configObj.write_wrapper_config2run_file(configName, line_cnt = i + 1) del configObj @@ -1287,8 +1281,7 @@ def unwrap_ion(self, pairs_same_starting_ranges_update, pairs_diff_starting_rang pairs1 = pairs_same_starting_ranges_update pairs2 = pairs_diff_starting_ranges_update - line_cnt = 0 - for p in pairs1+pairs2: + for i, p in enumerate(pairs1+pairs2): configName = os.path.join(self.config_path,'config_unwrap_ion_{}-{}'.format(p[0], p[1])) configObj = config(configName) configObj.configure(self) @@ -1327,9 +1320,8 @@ def unwrap_ion(self, pairs_same_starting_ranges_update, pairs_diff_starting_rang configObj.finalize() - line_cnt += 1 #line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt, self.numProcess) - line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt) + configObj.write_wrapper_config2run_file(configName, line_cnt = i + 1) del configObj @@ -1349,8 +1341,7 @@ def look_ion(self, pairs_same_starting_ranges_update, pairs_diff_starting_ranges pairs1 = pairs_same_starting_ranges_update pairs2 = pairs_diff_starting_ranges_update - line_cnt = 0 - for p in pairs1+pairs2: + for i, p in enumerate(pairs1+pairs2): configName = os.path.join(self.config_path,'config_look_ion_{}-{}'.format(p[0], p[1])) configObj = config(configName) configObj.configure(self) @@ -1397,9 +1388,8 @@ def look_ion(self, pairs_same_starting_ranges_update, pairs_diff_starting_ranges configObj.finalize() - line_cnt += 1 #line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt, self.numProcess) - line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt) + configObj.write_wrapper_config2run_file(configName, line_cnt = i + 1) del configObj @@ -1482,8 +1472,7 @@ def filtIon(self, pairs): ionParamUsrObj = ionParamUsr(self.param_ion) ionParamUsrObj.configure() - line_cnt = 0 - for p in pairs: + for i, p in enumerate(pairs): configName = os.path.join(self.config_path,'config_filtIon_{}_{}'.format(p[0], p[1])) configObj = config(configName) configObj.configure(self) @@ -1495,9 +1484,8 @@ def filtIon(self, pairs): configObj.filtIon('[Function-1]') configObj.finalize() - line_cnt += 1 #line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt, self.numProcess) - line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt) + configObj.write_wrapper_config2run_file(configName, line_cnt = i + 1) del configObj