From a25f80537a3d24eaf6936262ed6fc553c24e62b8 Mon Sep 17 00:00:00 2001 From: Bryan Marfito Date: Fri, 12 Mar 2021 23:36:45 +0800 Subject: [PATCH 01/20] Proposed changes to the fetchOrbit.py After running the currently modified fetchOrbits.py, I've found out that the downloaded precise orbits were based on the date that they were processed instead of the date that they can be used. I've also modified the code to prevent downloading of restituted orbit if the precise orbit is available. Pls. check if the changes I've made applies the orbit files to the Sentinel-1 SLC images properly. --- contrib/stack/topsStack/fetchOrbit.py | 30 ++++++++++++++++++++------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/contrib/stack/topsStack/fetchOrbit.py b/contrib/stack/topsStack/fetchOrbit.py index 9a11fa6a..400caccf 100755 --- a/contrib/stack/topsStack/fetchOrbit.py +++ b/contrib/stack/topsStack/fetchOrbit.py @@ -146,7 +146,16 @@ def fileToRange(fname): for spec in orbitMap: oType = spec[0] - url = server + spec[1] + str(fileTS.year).zfill(2) + '/' + str(fileTS.month).zfill(2) + \ + if oType == 'precise': + queryfmt3 = "%Y-%m-%d %H:%M:%S" + dt = datetime.datetime.strptime(str(fileTS), queryfmt3) + end_date = dt + datetime.timedelta(days=20) + fileTS1 = end_date.strftime(queryfmt3) + url = server + spec[1] + str(fileTS1[0:4]) + '/' + str(fileTS1[5:7]) + \ + '/' + str(fileTS1[8:10]) + '/' + + elif oType == 'restituted': + url = server + spec[1] + str(fileTS.year).zfill(2) + '/' + str(fileTS.month).zfill(2) + \ '/' + str(fileTS.day).zfill(2) + '/' success = False @@ -162,12 +171,17 @@ def fileToRange(fname): match = os.path.join(resulturl, result) if match is not None: success = True + except: pass - - if match is not None: - res = download_file(match, inps.outdir, session) - if res is False: - print('Failed to download URL: ', match) - else: - print('Failed to find {1} orbits for tref {0}'.format(fileTS, satName)) + + if success: + break + + if match is not None: + + res = download_file(match, inps.outdir, session) + if res is False: + print('Failed to download URL: ', match) + else: + print('Failed to find {1} orbits for tref {0}'.format(fileTS, satName)) From c8af76be104bd51f8a8e613e80e17d77c4b4ac30 Mon Sep 17 00:00:00 2001 From: Bryan Marfito Date: Sun, 14 Mar 2021 21:11:21 +0800 Subject: [PATCH 02/20] Update fetchOrbit.py --- contrib/stack/topsStack/fetchOrbit.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/contrib/stack/topsStack/fetchOrbit.py b/contrib/stack/topsStack/fetchOrbit.py index 400caccf..8ff595c0 100755 --- a/contrib/stack/topsStack/fetchOrbit.py +++ b/contrib/stack/topsStack/fetchOrbit.py @@ -152,11 +152,17 @@ def fileToRange(fname): end_date = dt + datetime.timedelta(days=20) fileTS1 = end_date.strftime(queryfmt3) url = server + spec[1] + str(fileTS1[0:4]) + '/' + str(fileTS1[5:7]) + \ - '/' + str(fileTS1[8:10]) + '/' + '/' + str(fileTS1[8:10]) + '/' elif oType == 'restituted': - url = server + spec[1] + str(fileTS.year).zfill(2) + '/' + str(fileTS.month).zfill(2) + \ - '/' + str(fileTS.day).zfill(2) + '/' + delta = datetime.timedelta(days=1) + timebef = (fileTS - delta).strftime(queryfmt) + timeaft = (fileTS + delta).strftime(queryfmt) + url = (server + spec[1] + str(fileTS.year).zfill(2) + '/' + str(fileTS.month).zfill(2) + \ + '/' + str(fileTS.day).zfill(2) + '/' + '?validity_start={0}..{1}&sentinel1__mission={2}'.\ + format(timebef, timeaft,satName)) + + success = False match = None @@ -164,22 +170,20 @@ def fileToRange(fname): try: r = session.get(url, verify=False) r.raise_for_status() - parser = MyHTMLParser(satName, url) parser.feed(r.text) for resulturl, result in parser.fileList: match = os.path.join(resulturl, result) if match is not None: success = True - except: pass - + if success: break - + if match is not None: - + res = download_file(match, inps.outdir, session) if res is False: print('Failed to download URL: ', match) From 83f363d0c57787af4914039fc94dc803aceffeb6 Mon Sep 17 00:00:00 2001 From: Bryan Marfito Date: Sun, 14 Mar 2021 21:29:13 +0800 Subject: [PATCH 03/20] Update fetchOrbit.py --- contrib/stack/topsStack/fetchOrbit.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/stack/topsStack/fetchOrbit.py b/contrib/stack/topsStack/fetchOrbit.py index 8ff595c0..76b78cc3 100755 --- a/contrib/stack/topsStack/fetchOrbit.py +++ b/contrib/stack/topsStack/fetchOrbit.py @@ -158,9 +158,9 @@ def fileToRange(fname): delta = datetime.timedelta(days=1) timebef = (fileTS - delta).strftime(queryfmt) timeaft = (fileTS + delta).strftime(queryfmt) - url = (server + spec[1] + str(fileTS.year).zfill(2) + '/' + str(fileTS.month).zfill(2) + \ + url = server + spec[1] + str(fileTS.year).zfill(2) + '/' + str(fileTS.month).zfill(2) + \ '/' + str(fileTS.day).zfill(2) + '/' + '?validity_start={0}..{1}&sentinel1__mission={2}'.\ - format(timebef, timeaft,satName)) + format(timebef, timeaft,satName) From a4cf6f5b585c9b3ad190112922e76b9e2be43cee Mon Sep 17 00:00:00 2001 From: Bryan Marfito Date: Wed, 17 Mar 2021 21:20:52 +0800 Subject: [PATCH 04/20] Update fetchOrbit.py --- contrib/stack/topsStack/fetchOrbit.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/contrib/stack/topsStack/fetchOrbit.py b/contrib/stack/topsStack/fetchOrbit.py index 76b78cc3..8006dd4c 100755 --- a/contrib/stack/topsStack/fetchOrbit.py +++ b/contrib/stack/topsStack/fetchOrbit.py @@ -1,5 +1,7 @@ #!/usr/bin/env python3 +#!/usr/bin/env python3 + import numpy as np import re import requests @@ -159,8 +161,7 @@ def fileToRange(fname): timebef = (fileTS - delta).strftime(queryfmt) timeaft = (fileTS + delta).strftime(queryfmt) url = server + spec[1] + str(fileTS.year).zfill(2) + '/' + str(fileTS.month).zfill(2) + \ - '/' + str(fileTS.day).zfill(2) + '/' + '?validity_start={0}..{1}&sentinel1__mission={2}'.\ - format(timebef, timeaft,satName) + '/' + str(fileTS.day).zfill(2) +'/' @@ -173,9 +174,16 @@ def fileToRange(fname): parser = MyHTMLParser(satName, url) parser.feed(r.text) for resulturl, result in parser.fileList: - match = os.path.join(resulturl, result) - if match is not None: - success = True + if oType == 'precise': + match = os.path.join(resulturl, result) + elif oType == "restituted": + tbef, taft, mission = fileToRange(os.path.basename(result)) + if (tbef <= fileTSStart) and (taft >= fileTS): + datestr2 = FileToTimeStamp(result)[0].strftime(queryfmt2) + match = url + result + + if match is not None: + success = True except: pass From e5e0a4560d1f6a472eac05c39682614af4b843df Mon Sep 17 00:00:00 2001 From: Bryan Marfito Date: Wed, 17 Mar 2021 21:36:43 +0800 Subject: [PATCH 05/20] Update fetchOrbit.py --- contrib/stack/topsStack/fetchOrbit.py | 1 - 1 file changed, 1 deletion(-) diff --git a/contrib/stack/topsStack/fetchOrbit.py b/contrib/stack/topsStack/fetchOrbit.py index 8006dd4c..ac3a6a4d 100755 --- a/contrib/stack/topsStack/fetchOrbit.py +++ b/contrib/stack/topsStack/fetchOrbit.py @@ -179,7 +179,6 @@ def fileToRange(fname): elif oType == "restituted": tbef, taft, mission = fileToRange(os.path.basename(result)) if (tbef <= fileTSStart) and (taft >= fileTS): - datestr2 = FileToTimeStamp(result)[0].strftime(queryfmt2) match = url + result if match is not None: From 198b23aa3fa2233078fd6c5cd72d6f4c647ffcd1 Mon Sep 17 00:00:00 2001 From: Bryan Marfito Date: Wed, 17 Mar 2021 22:00:21 +0800 Subject: [PATCH 06/20] Update fetchOrbit.py --- contrib/stack/topsStack/fetchOrbit.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/contrib/stack/topsStack/fetchOrbit.py b/contrib/stack/topsStack/fetchOrbit.py index ac3a6a4d..767bf0e4 100755 --- a/contrib/stack/topsStack/fetchOrbit.py +++ b/contrib/stack/topsStack/fetchOrbit.py @@ -149,22 +149,16 @@ def fileToRange(fname): oType = spec[0] if oType == 'precise': + end_date = fileTS + datetime.timedelta(days=20) queryfmt3 = "%Y-%m-%d %H:%M:%S" - dt = datetime.datetime.strptime(str(fileTS), queryfmt3) - end_date = dt + datetime.timedelta(days=20) fileTS1 = end_date.strftime(queryfmt3) url = server + spec[1] + str(fileTS1[0:4]) + '/' + str(fileTS1[5:7]) + \ '/' + str(fileTS1[8:10]) + '/' elif oType == 'restituted': - delta = datetime.timedelta(days=1) - timebef = (fileTS - delta).strftime(queryfmt) - timeaft = (fileTS + delta).strftime(queryfmt) url = server + spec[1] + str(fileTS.year).zfill(2) + '/' + str(fileTS.month).zfill(2) + \ '/' + str(fileTS.day).zfill(2) +'/' - - success = False match = None From a8e3e2c175c27e6200027d7523c7a1f838d36d03 Mon Sep 17 00:00:00 2001 From: Bryan Marfito Date: Wed, 17 Mar 2021 22:04:13 +0800 Subject: [PATCH 07/20] Update fetchOrbit.py --- contrib/stack/topsStack/fetchOrbit.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/contrib/stack/topsStack/fetchOrbit.py b/contrib/stack/topsStack/fetchOrbit.py index 767bf0e4..efd0c0be 100755 --- a/contrib/stack/topsStack/fetchOrbit.py +++ b/contrib/stack/topsStack/fetchOrbit.py @@ -150,10 +150,8 @@ def fileToRange(fname): if oType == 'precise': end_date = fileTS + datetime.timedelta(days=20) - queryfmt3 = "%Y-%m-%d %H:%M:%S" - fileTS1 = end_date.strftime(queryfmt3) - url = server + spec[1] + str(fileTS1[0:4]) + '/' + str(fileTS1[5:7]) + \ - '/' + str(fileTS1[8:10]) + '/' + url = server + spec[1] + str(end_date.year).zfill(2) + '/' + str(end_date.month).zfill(2) + \ + '/' + str(end_date.day).zfill(2) + '/' elif oType == 'restituted': url = server + spec[1] + str(fileTS.year).zfill(2) + '/' + str(fileTS.month).zfill(2) + \ From ad2729be000b150b4d72667ff408833433357992 Mon Sep 17 00:00:00 2001 From: Bryan Marfito Date: Sun, 21 Mar 2021 20:12:24 +0800 Subject: [PATCH 08/20] Update fetchOrbit.py --- contrib/stack/topsStack/fetchOrbit.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/contrib/stack/topsStack/fetchOrbit.py b/contrib/stack/topsStack/fetchOrbit.py index efd0c0be..b4269736 100755 --- a/contrib/stack/topsStack/fetchOrbit.py +++ b/contrib/stack/topsStack/fetchOrbit.py @@ -1,7 +1,5 @@ #!/usr/bin/env python3 -#!/usr/bin/env python3 - import numpy as np import re import requests From 415a89365c2b0b6aaddd53dd299aa38c04e66b9f Mon Sep 17 00:00:00 2001 From: Bryan Marfito Date: Sun, 21 Mar 2021 20:13:15 +0800 Subject: [PATCH 09/20] Update fetchOrbit.py --- contrib/stack/topsStack/fetchOrbit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/stack/topsStack/fetchOrbit.py b/contrib/stack/topsStack/fetchOrbit.py index b4269736..820bb6b9 100755 --- a/contrib/stack/topsStack/fetchOrbit.py +++ b/contrib/stack/topsStack/fetchOrbit.py @@ -166,7 +166,7 @@ def fileToRange(fname): for resulturl, result in parser.fileList: if oType == 'precise': match = os.path.join(resulturl, result) - elif oType == "restituted": + elif oType == 'restituted': tbef, taft, mission = fileToRange(os.path.basename(result)) if (tbef <= fileTSStart) and (taft >= fileTS): match = url + result From 7c9f061f590574808c6cf37df63c8749ecc977c1 Mon Sep 17 00:00:00 2001 From: Bryan Marfito Date: Tue, 23 Mar 2021 19:01:01 +0800 Subject: [PATCH 10/20] Update fetchOrbit.py --- contrib/stack/topsStack/fetchOrbit.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/stack/topsStack/fetchOrbit.py b/contrib/stack/topsStack/fetchOrbit.py index 820bb6b9..a52f990d 100755 --- a/contrib/stack/topsStack/fetchOrbit.py +++ b/contrib/stack/topsStack/fetchOrbit.py @@ -166,10 +166,10 @@ def fileToRange(fname): for resulturl, result in parser.fileList: if oType == 'precise': match = os.path.join(resulturl, result) - elif oType == 'restituted': + elif oType == "restituted": tbef, taft, mission = fileToRange(os.path.basename(result)) if (tbef <= fileTSStart) and (taft >= fileTS): - match = url + result + match = os.path.join(resulturl, result) if match is not None: success = True From f1187fd935a41eb2ded1b77bfa73b214a3977706 Mon Sep 17 00:00:00 2001 From: Bryan Marfito Date: Wed, 24 Mar 2021 06:21:03 +0800 Subject: [PATCH 11/20] Update contrib/stack/topsStack/fetchOrbit.py Co-authored-by: Ryan Burns --- contrib/stack/topsStack/fetchOrbit.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/contrib/stack/topsStack/fetchOrbit.py b/contrib/stack/topsStack/fetchOrbit.py index a52f990d..3b31f6bf 100755 --- a/contrib/stack/topsStack/fetchOrbit.py +++ b/contrib/stack/topsStack/fetchOrbit.py @@ -148,12 +148,12 @@ def fileToRange(fname): if oType == 'precise': end_date = fileTS + datetime.timedelta(days=20) - url = server + spec[1] + str(end_date.year).zfill(2) + '/' + str(end_date.month).zfill(2) + \ - '/' + str(end_date.day).zfill(2) + '/' - elif oType == 'restituted': - url = server + spec[1] + str(fileTS.year).zfill(2) + '/' + str(fileTS.month).zfill(2) + \ - '/' + str(fileTS.day).zfill(2) +'/' + end_date = fileTS + else: + throw ValueError("Unexpected orbit type: '" + oType + "'") + url = server + spec[1] + str(end_date.year).zfill(2) + '/' + str(end_date.month).zfill(2) + \ + '/' + str(end_date.day).zfill(2) + '/' success = False match = None From 3499b3f09bc65020a44e6f25d45ad1ba0c3ccb5e Mon Sep 17 00:00:00 2001 From: Bryan Marfito Date: Wed, 24 Mar 2021 06:21:44 +0800 Subject: [PATCH 12/20] Update fetchOrbit.py --- contrib/stack/topsStack/fetchOrbit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/stack/topsStack/fetchOrbit.py b/contrib/stack/topsStack/fetchOrbit.py index 3b31f6bf..a8e065ee 100755 --- a/contrib/stack/topsStack/fetchOrbit.py +++ b/contrib/stack/topsStack/fetchOrbit.py @@ -151,7 +151,7 @@ def fileToRange(fname): elif oType == 'restituted': end_date = fileTS else: - throw ValueError("Unexpected orbit type: '" + oType + "'") + raise ValueError("Unexpected orbit type: '" + oType + "'") url = server + spec[1] + str(end_date.year).zfill(2) + '/' + str(end_date.month).zfill(2) + \ '/' + str(end_date.day).zfill(2) + '/' From a10cba9f939d3509a04fb585974ab394c6ca36a1 Mon Sep 17 00:00:00 2001 From: Bryan Marfito Date: Wed, 24 Mar 2021 08:07:19 +0800 Subject: [PATCH 13/20] Update fetchOrbit.py --- contrib/stack/topsStack/fetchOrbit.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/contrib/stack/topsStack/fetchOrbit.py b/contrib/stack/topsStack/fetchOrbit.py index a8e065ee..51e13f6c 100755 --- a/contrib/stack/topsStack/fetchOrbit.py +++ b/contrib/stack/topsStack/fetchOrbit.py @@ -152,17 +152,20 @@ def fileToRange(fname): end_date = fileTS else: raise ValueError("Unexpected orbit type: '" + oType + "'") + end_date2 = end_date + datetime.timedelta(days=1) url = server + spec[1] + str(end_date.year).zfill(2) + '/' + str(end_date.month).zfill(2) + \ - '/' + str(end_date.day).zfill(2) + '/' + '/' + str(end_date.day).zfill(2) + '/', server + spec[1] + str(end_date2.year).zfill(2) + '/' + str(end_date2.month).zfill(2) + \ + '/' + str(end_date2.day).zfill(2) + '/' success = False match = None try: - r = session.get(url, verify=False) - r.raise_for_status() - parser = MyHTMLParser(satName, url) - parser.feed(r.text) + for url1 in url: + r = session.get(url1, verify=False) + r.raise_for_status() + parser = MyHTMLParser(satName, url1) + parser.feed(r.text) for resulturl, result in parser.fileList: if oType == 'precise': match = os.path.join(resulturl, result) From f67a4cfbaa7ba60d912b86724e50be8349aff39f Mon Sep 17 00:00:00 2001 From: Bryan Marfito Date: Wed, 24 Mar 2021 09:48:09 +0800 Subject: [PATCH 14/20] Update fetchOrbit.py --- contrib/stack/topsStack/fetchOrbit.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/contrib/stack/topsStack/fetchOrbit.py b/contrib/stack/topsStack/fetchOrbit.py index 51e13f6c..60f532fb 100755 --- a/contrib/stack/topsStack/fetchOrbit.py +++ b/contrib/stack/topsStack/fetchOrbit.py @@ -161,18 +161,20 @@ def fileToRange(fname): match = None try: + for url1 in url: r = session.get(url1, verify=False) r.raise_for_status() parser = MyHTMLParser(satName, url1) - parser.feed(r.text) - for resulturl, result in parser.fileList: - if oType == 'precise': - match = os.path.join(resulturl, result) - elif oType == "restituted": - tbef, taft, mission = fileToRange(os.path.basename(result)) - if (tbef <= fileTSStart) and (taft >= fileTS): + parser.feed(r.text + "\n") + + for resulturl, result in parser.fileList: + if oType == 'precise': match = os.path.join(resulturl, result) + elif oType == "restituted": + tbef, taft, mission = fileToRange(os.path.basename(result)) + if (tbef <= fileTSStart) and (taft >= fileTS): + match = os.path.join(resulturl, result) if match is not None: success = True From 92ecd7d5edbd433d27666911a06c0a4bb25f596a Mon Sep 17 00:00:00 2001 From: Bryan Marfito Date: Wed, 24 Mar 2021 09:57:01 +0800 Subject: [PATCH 15/20] Update fetchOrbit.py --- contrib/stack/topsStack/fetchOrbit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/stack/topsStack/fetchOrbit.py b/contrib/stack/topsStack/fetchOrbit.py index 60f532fb..e41a7ade 100755 --- a/contrib/stack/topsStack/fetchOrbit.py +++ b/contrib/stack/topsStack/fetchOrbit.py @@ -171,7 +171,7 @@ def fileToRange(fname): for resulturl, result in parser.fileList: if oType == 'precise': match = os.path.join(resulturl, result) - elif oType == "restituted": + elif oType == 'restituted': tbef, taft, mission = fileToRange(os.path.basename(result)) if (tbef <= fileTSStart) and (taft >= fileTS): match = os.path.join(resulturl, result) From ced19384721256f9c69f738c7be9a3e6ebd5ee3d Mon Sep 17 00:00:00 2001 From: Bryan Marfito Date: Wed, 24 Mar 2021 09:57:20 +0800 Subject: [PATCH 16/20] Update fetchOrbit.py --- contrib/stack/topsStack/fetchOrbit.py | 1 - 1 file changed, 1 deletion(-) diff --git a/contrib/stack/topsStack/fetchOrbit.py b/contrib/stack/topsStack/fetchOrbit.py index e41a7ade..4c7791e3 100755 --- a/contrib/stack/topsStack/fetchOrbit.py +++ b/contrib/stack/topsStack/fetchOrbit.py @@ -161,7 +161,6 @@ def fileToRange(fname): match = None try: - for url1 in url: r = session.get(url1, verify=False) r.raise_for_status() From 32b4ac58be3f7cdb74fa3b51bf9160bf5009ec7c Mon Sep 17 00:00:00 2001 From: Bryan Marfito Date: Wed, 24 Mar 2021 14:13:54 +0800 Subject: [PATCH 17/20] Update contrib/stack/topsStack/fetchOrbit.py Co-authored-by: Ryan Burns --- contrib/stack/topsStack/fetchOrbit.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/contrib/stack/topsStack/fetchOrbit.py b/contrib/stack/topsStack/fetchOrbit.py index 4c7791e3..082469d9 100755 --- a/contrib/stack/topsStack/fetchOrbit.py +++ b/contrib/stack/topsStack/fetchOrbit.py @@ -153,9 +153,7 @@ def fileToRange(fname): else: raise ValueError("Unexpected orbit type: '" + oType + "'") end_date2 = end_date + datetime.timedelta(days=1) - url = server + spec[1] + str(end_date.year).zfill(2) + '/' + str(end_date.month).zfill(2) + \ - '/' + str(end_date.day).zfill(2) + '/', server + spec[1] + str(end_date2.year).zfill(2) + '/' + str(end_date2.month).zfill(2) + \ - '/' + str(end_date2.day).zfill(2) + '/' + url = (server + spec[1] + end_date.strftime("%Y/%m/%d/") for end_date in (end_date, end_date2)) success = False match = None From b5c1c6420a017644c0dc743d601e487980575db8 Mon Sep 17 00:00:00 2001 From: Bryan Marfito Date: Wed, 24 Mar 2021 14:14:33 +0800 Subject: [PATCH 18/20] Update fetchOrbit.py --- contrib/stack/topsStack/fetchOrbit.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/contrib/stack/topsStack/fetchOrbit.py b/contrib/stack/topsStack/fetchOrbit.py index 082469d9..afd558a5 100755 --- a/contrib/stack/topsStack/fetchOrbit.py +++ b/contrib/stack/topsStack/fetchOrbit.py @@ -153,17 +153,17 @@ def fileToRange(fname): else: raise ValueError("Unexpected orbit type: '" + oType + "'") end_date2 = end_date + datetime.timedelta(days=1) - url = (server + spec[1] + end_date.strftime("%Y/%m/%d/") for end_date in (end_date, end_date2)) + urls = (server + spec[1] + end_date.strftime("%Y/%m/%d/") for end_date in (end_date, end_date2)) success = False match = None try: - for url1 in url: - r = session.get(url1, verify=False) + for url in urls: + r = session.get(url, verify=False) r.raise_for_status() - parser = MyHTMLParser(satName, url1) - parser.feed(r.text + "\n") + parser = MyHTMLParser(satName, url) + parser.feed(r.text) for resulturl, result in parser.fileList: if oType == 'precise': From 41721e3d1da6c5183a9cd93245f4821771a2f258 Mon Sep 17 00:00:00 2001 From: Bryan Marfito Date: Thu, 25 Mar 2021 06:32:37 +0800 Subject: [PATCH 19/20] Update fetchOrbit.py --- contrib/stack/topsStack/fetchOrbit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/stack/topsStack/fetchOrbit.py b/contrib/stack/topsStack/fetchOrbit.py index afd558a5..6176750c 100755 --- a/contrib/stack/topsStack/fetchOrbit.py +++ b/contrib/stack/topsStack/fetchOrbit.py @@ -147,7 +147,7 @@ def fileToRange(fname): oType = spec[0] if oType == 'precise': - end_date = fileTS + datetime.timedelta(days=20) + end_date = fileTS + datetime.timedelta(days=19) elif oType == 'restituted': end_date = fileTS else: From f3d38d4135d140bc41f2c224e25da4ba80b9417a Mon Sep 17 00:00:00 2001 From: Bryan Marfito Date: Thu, 25 Mar 2021 08:16:42 +0800 Subject: [PATCH 20/20] Update fetchOrbit.py --- contrib/stack/topsStack/fetchOrbit.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/contrib/stack/topsStack/fetchOrbit.py b/contrib/stack/topsStack/fetchOrbit.py index 6176750c..7e7192fb 100755 --- a/contrib/stack/topsStack/fetchOrbit.py +++ b/contrib/stack/topsStack/fetchOrbit.py @@ -147,7 +147,7 @@ def fileToRange(fname): oType = spec[0] if oType == 'precise': - end_date = fileTS + datetime.timedelta(days=19) + end_date = fileTS + datetime.timedelta(days=20) elif oType == 'restituted': end_date = fileTS else: @@ -159,6 +159,7 @@ def fileToRange(fname): match = None try: + for url in urls: r = session.get(url, verify=False) r.raise_for_status() @@ -166,12 +167,9 @@ def fileToRange(fname): parser.feed(r.text) for resulturl, result in parser.fileList: - if oType == 'precise': + tbef, taft, mission = fileToRange(os.path.basename(result)) + if (tbef <= fileTSStart) and (taft >= fileTS): match = os.path.join(resulturl, result) - elif oType == 'restituted': - tbef, taft, mission = fileToRange(os.path.basename(result)) - if (tbef <= fileTSStart) and (taft >= fileTS): - match = os.path.join(resulturl, result) if match is not None: success = True