From 36910f270783c8654609a894f602a2e56f070e8a Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Wed, 14 Aug 2024 16:42:17 -0400 Subject: [PATCH 1/6] DOC: updated docstrings Updated the F10.7 docstrings to be more correct. --- pysatSpaceWeather/instruments/sw_f107.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pysatSpaceWeather/instruments/sw_f107.py b/pysatSpaceWeather/instruments/sw_f107.py index 0bcf106..e2ec995 100644 --- a/pysatSpaceWeather/instruments/sw_f107.py +++ b/pysatSpaceWeather/instruments/sw_f107.py @@ -95,7 +95,7 @@ # Dict keyed by inst_id that lists supported tags and a good day of test data # generate todays date to support loading forecast data -now = dt.datetime.utcnow() +now = dt.datetime.now(tz=dt.timezone.utc) today = dt.datetime(now.year, now.month, now.day) tomorrow = today + dt.timedelta(days=1) @@ -157,7 +157,7 @@ def load(fnames, tag='', inst_id=''): tag : str Instrument tag. (default='') inst_id : str - Instrument ID, not used. (default='') + Instrument ID. (default='') Returns ------- @@ -263,8 +263,8 @@ def list_files(tag='', inst_id='', data_path='', format_str=None): Returns ------- - out_files : pysat._files.Files - A class containing the verified available files + out_files : pds.Series + A Series containing the verified available files Note ---- From 7725299c1b7ad59872c079f0afb8eb3f09b5b40c Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Wed, 14 Aug 2024 16:42:54 -0400 Subject: [PATCH 2/6] ENH: added NoRP methods Added Instrument support methods for Nobeyama Radio Polarimeters (NoRP) data. --- pysatSpaceWeather/instruments/methods/norp.py | 225 ++++++++++++++++++ 1 file changed, 225 insertions(+) create mode 100644 pysatSpaceWeather/instruments/methods/norp.py diff --git a/pysatSpaceWeather/instruments/methods/norp.py b/pysatSpaceWeather/instruments/methods/norp.py new file mode 100644 index 0000000..f6a8bb8 --- /dev/null +++ b/pysatSpaceWeather/instruments/methods/norp.py @@ -0,0 +1,225 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*-. +# Full license can be found in License.md +# Full author list can be found in .zenodo.json file +# DOI:10.5281/zenodo.3986138 +# ---------------------------------------------------------------------------- +"""Supports data from the Nobeyama Radio Polarimeters.""" + +import datetime as dt +import numpy as np +import os +import pandas as pds +import requests + +import pysat + + +def acknowledgements(): + """Define the acknowledgements for NoRP data. + + Returns + ------- + ackn : str + Acknowledgements associated with the appropriate NoRP name and tag. + + """ + ackn = ''.join(['The Nobeyama Radio Polarimeters (NoRP) are operated by ', + 'Solar Science Observatory, a branch of National ', + 'Astronomical Observatory of Japan, and their observing ', + 'data are verified scientifically by the consortium for ', + 'NoRP scientific operations.', + '\nFor questions regarding the data please contact ', + 'solar_helpdesk@ml.nao.ac.jp']) + + return ackn + + +def references(name, tag): + """Define the references for NoRP data. + + Parameters + ---------- + name : str + Instrument name for the NoRP data. + tag : str + Instrument tag for the NoRP data. + + Returns + ------- + refs : str + Reference string associated with the appropriate F10.7 tag. + + """ + refs = {'rf': {'daily': "\n".join([ + ''.join(['Shimojo and Iwai "Over seven decades of solar microwave ', + 'data obtained with Toyokawa and Nobeyama Radio Polarimeters', + '", GDJ, 10, 114-129 (2023)']), + ''.join(['Nakajima et al. "The Radiometer and Polarimeters at 80, 35,', + ' and 17 GHz for Solar Observations at Nobeyama", PASJ, 37,', + ' 163 (1985)']), + ''.join(['Torii et al. "Full-Automatic Radiopolarimeters for Solar ', + 'Patrol at Microwave Frequencies", Proc. of the Res. Inst. ', + 'of Atmospherics, Nagoya Univ., 26, 129 (1979)']), + ''.join(['Shibasaki et al. "Solar Radio Data Acquisition and ', + 'Communication System (SORDACS) of Toyokawa Observatory", ', + 'Proc. of the Res. Inst. of Atmospherics, Nagoya Univ., 26, ', + '117 (1979)']), + 'Tanaka, "Toyokawa Observatory", Solar Physics, 1, 2, 295 (1967)', + ''.join(['Tsuchiya and Nagase "Atmospheric Absorption in Microwave ', + 'Solar Observation and Solar Flux Measurement at 17 Gc/s", ', + 'PASJ, 17, 86 (1965)']), + ''.join(['Tanaka and Kakinuma. "EQUIPMENT FOR THE OBSERVATION OF SOLAR', + ' RADIO EMISSION AT 9400, 3750, 2000 AND 1000 Mc/s", Proc. ', + 'of the Res. Inst. of Atmospherics, Nagoya Univ., 4, 60 ', + '(1957)']), + ''.join(['Tanaka et al. "EQUIPMENT FOR THE OBSERVATION OF SOLAR NOISE ', + 'AT 3,750 MC", Proc. of the Res. Inst. of Atmospherics, ', + 'Nagoya Univ., 1, 71 (1953)'])])}} + + return refs[name][tag] + + +def daily_rf_downloads(data_path, mock_download_dir=None, start=None, + stop=None): + """Download LASP 96-hour prediction data. + + Parameters + ---------- + data_path : str + Path to data directory. + mock_download_dir : str or NoneType + Local directory with downloaded files or None. If not None, will + process any files with the correct name and date (following the local + file prefix and date format) as if they were downloaded (default=None) + + Raises + ------ + IOError + If the data link has an unexpected format or an unknown mock download + directory is supplied. + + Note + ---- + Saves data in month-long files + + """ + # Initalize the output information + times = list() + data_dict = dict() + + # Set the file name + fname = 'TYKW-NoRP_dailyflux.txt' + + if mock_download_dir is None: + # Set the remote data variables + url = '/'.join(['https://solar.nro.nao.ac.jp/norp/data/daily', fname]) + + # Download the webpage + req = requests.get(url) + + # Test to see if the file was found on the server + if req.text.find('not found on this server') > 0: + pysat.logger.warning(''.join(['NoRP daily flux file not found on ', + 'server: ', url])) + raw_txt = None + else: + raw_txt = req.text + else: + # If a mock download directory was supplied, test to see it exists + if mock_download_dir is not None: + if not os.path.isdir(mock_download_dir): + raise IOError('file location is not a directory: {:}'.format( + mock_download_dir)) + + # Get the data from the mock download directory + url = os.path.join(mock_download_dir, fname) + if os.path.isfile(url): + with open(url, 'r') as fpin: + raw_txt = fpin.read() + else: + pysat.logger.warning(''.join(['NoRP daily flux file not found in', + 'the local directory: ', url, + ", data may have been saved to an ", + "unexpected filename"])) + raw_txt = None + + if raw_txt is not None: + # Split the text to get the header lines + file_lines = raw_txt.split('\n')[:2] + + # If needed, set or adjust the start and end times + line_cols = file_lines[0].split() + file_start = dt.datetime.strptime(line_cols[-3], '(%Y-%m-%d') + file_stop = dt.datetime.strptime(line_cols[-1], '%Y-%m-%d)') + + # Evaluate the file start time + if start is None or start < file_start: + start = file_start + elif start.day > 1: + # Set the start time to be the start of the month + start = dt.datetime(start.year, start.month, 1) + + # Evaluate the file stop time + if stop is None or stop < file_stop: + stop = file_stop + elif stop.day < 31: + # Set the stop time to be the end of the month + month_end = stop + dt.timedelta(days=1) + while month_end.month == stop.month: + stop = dt.datetime(month_end.year, month_end.month, + month_end.day) + month_end += dt.timedelta(days=1) + + # Set the data columns + data_cols = [col.replace(" ", "_") for col in file_lines[1].split(',')] + for col in data_cols[1:]: + data_dict[col] = list() + + # Split the text to get the file line for the desired period + start_txt = raw_txt.split(start.strftime('"%Y-%m-%d"'))[1] + stop_txt = ''.join([start.strftime('"%Y-%m-%d"'), start_txt]).split( + stop.strftime('"%Y-%m-%d"')) + file_lines = stop_txt[0].split('\n')[:-1] + if len(stop_txt) > 1: + file_lines.append(''.join([stop.strftime('"%Y-%m-%d"'), + stop_txt[1]]).split('\n')[0]) + + # Format the data for the desired time period + for line in file_lines: + # Split the line on comma + line_cols = line.split(',') + + if len(line_cols) != len(data_cols): + raise IOError(''.join(['unexpected line encountered in file ', + 'retrieved from ', url, ':\n', line])) + + # Format the time and values + times.append(dt.datetime.strptime(line_cols[0], '"%Y-%m-%d"')) + for i, col in enumerate(data_cols[1:]): + if line_cols[i + 1].lower().find('nan') == 0: + data_dict[col].append(np.nan) + else: + data_dict[col].append(np.float64(line_cols[i + 1])) + + # Re-cast the data as a pandas DataFrame + data = pds.DataFrame(data_dict, index=times) + + # Write out the files using a monthly cadance + file_base = '_'.join(['norp', 'rf', 'daily', '%Y-%m.txt']) + + while start < stop: + # Set the output file name + file_name = os.path.join(data_path, start.strftime(file_base)) + + # Downselect the output data + file_data = data[start:start + pds.DateOffset(months=1) + - dt.timedelta(microseconds=1)] + + # Save the output data to file + file_data.to_csv(file_name) + + # Cycle the time + start += pds.DateOffset(months=1) + + return From b3cb0dee39248c13188c6242d25ab96b0c863f75 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Wed, 14 Aug 2024 16:43:41 -0400 Subject: [PATCH 3/6] ENH: added NoRP solar radio flux data Added a new instrument for NoRP solar radio flux data. --- pysatSpaceWeather/instruments/__init__.py | 8 +- .../instruments/methods/__init__.py | 1 + pysatSpaceWeather/instruments/norp_rf.py | 223 ++++++++++++++++++ 3 files changed, 228 insertions(+), 4 deletions(-) create mode 100644 pysatSpaceWeather/instruments/norp_rf.py diff --git a/pysatSpaceWeather/instruments/__init__.py b/pysatSpaceWeather/instruments/__init__.py index efa9202..fe819be 100644 --- a/pysatSpaceWeather/instruments/__init__.py +++ b/pysatSpaceWeather/instruments/__init__.py @@ -1,9 +1,9 @@ from pysatSpaceWeather.instruments import methods # noqa F401 -__all__ = ['ace_epam', 'ace_mag', 'ace_sis', 'ace_swepam', 'sw_ae', 'sw_al', - 'sw_au', 'sw_ap', 'sw_apo', 'sw_cp', 'sw_dst', 'sw_f107', 'sw_flare', - 'sw_hpo', 'sw_kp', 'sw_mgii', 'sw_polarcap', 'sw_sbfield', 'sw_ssn', - 'sw_stormprob'] +__all__ = ['ace_epam', 'ace_mag', 'ace_sis', 'ace_swepam', 'norp_rf', 'sw_ae', + 'sw_al', 'sw_au', 'sw_ap', 'sw_apo', 'sw_cp', 'sw_dst', 'sw_f107', + 'sw_flare', 'sw_hpo', 'sw_kp', 'sw_mgii', 'sw_polarcap', + 'sw_sbfield', 'sw_ssn', 'sw_stormprob'] for inst in __all__: exec("from pysatSpaceWeather.instruments import {inst}".format(inst=inst)) diff --git a/pysatSpaceWeather/instruments/methods/__init__.py b/pysatSpaceWeather/instruments/methods/__init__.py index 28ada58..c2e258f 100644 --- a/pysatSpaceWeather/instruments/methods/__init__.py +++ b/pysatSpaceWeather/instruments/methods/__init__.py @@ -4,6 +4,7 @@ from pysatSpaceWeather.instruments.methods import f107 # noqa F401 from pysatSpaceWeather.instruments.methods import general # noqa F401 from pysatSpaceWeather.instruments.methods import gfz # noqa F401 +from pysatSpaceWeather.instruments.methods import norp # noqa F401 from pysatSpaceWeather.instruments.methods import kp_ap # noqa F401 from pysatSpaceWeather.instruments.methods import lasp # noqa F401 from pysatSpaceWeather.instruments.methods import lisird # noqa F401 diff --git a/pysatSpaceWeather/instruments/norp_rf.py b/pysatSpaceWeather/instruments/norp_rf.py new file mode 100644 index 0000000..fc32a0a --- /dev/null +++ b/pysatSpaceWeather/instruments/norp_rf.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*-. +# Full license can be found in License.md +# Full author list can be found in .zenodo.json file +# DOI:10.5281/zenodo.3986138 +# ---------------------------------------------------------------------------- +"""Supports solar radio flux values from the Nobeyama Radio Polarimeters. + +Properties +---------- +platform + 'norp' +name + 'rf' +tag + - 'daily' Daily solar flux values from 1951-11-01 onward +inst_id + - None supported + +Examples +-------- +Download and load all of the daily radio flux data. +:: + + rf = pysat.Instrument('norp', 'rf', tag='daily') + rf.download(start=dt.datetime(1951, 11, 1), stop=rf.today()) + rf.load(date=dt.datetime(1951, 11, 1), end_date=rf.today()) + +""" + +import datetime as dt +import numpy as np +import pandas as pds + +import pysat + +from pysatSpaceWeather.instruments import methods + +# ---------------------------------------------------------------------------- +# Instrument attributes + +platform = 'norp' +name = 'rf' +tags = {'daily': 'Daily solar flux values'} +inst_ids = {'': [tag for tag in tags.keys()]} + +# ---------------------------------------------------------------------------- +# Instrument test attributes + +_test_dates = {'': {'daily': dt.datetime(1951, 11, 1)}} + +# ---------------------------------------------------------------------------- +# Instrument methods + +preprocess = methods.general.preprocess + + +def init(self): + """Initialize the Instrument object with instrument specific values.""" + + # Set the required Instrument attributes + self.acknowledgements = methods.norp.acknowledgements() + self.references = methods.norp.references(self.name, self.tag) + pysat.logger.info(self.acknowledgements) + + return + + +def clean(self): + """Clean the solar radio flux, empty function as this is not necessary.""" + + return + + +# ---------------------------------------------------------------------------- +# Instrument functions + + +def load(fnames, tag='', inst_id=''): + """Load NoRP solar radio flux files. + + Parameters + ---------- + fnames : pandas.Series + Series of filenames. + tag : str + Instrument tag. (default='') + inst_id : str + Instrument ID, not used. (default='') + + Returns + ------- + data : pandas.DataFrame + Object containing satellite data. + meta : pysat.Meta + Object containing metadata such as column names and units. + + See Also + -------- + pysat.instruments.methods.general.load_csv_data + + Note + ---- + Called by pysat. Not intended for direct use by user. + + """ + + # Get the desired file dates and file names from the daily indexed list + file_dates = list() + if tag in ['daily']: + unique_files = list() + for fname in fnames: + file_dates.append(dt.datetime.strptime(fname[-10:], '%Y-%m-%d')) + if fname[0:-11] not in unique_files: + unique_files.append(fname[0:-11]) + fnames = unique_files + + # Load the CSV data files + data = pysat.instruments.methods.general.load_csv_data( + fnames, read_csv_kwargs={"index_col": 0, "parse_dates": True}) + + # If there is a date range, downselect here + if len(file_dates) > 0: + idx, = np.where((data.index >= min(file_dates)) + & (data.index < max(file_dates) + dt.timedelta(days=1))) + data = data.iloc[idx, :] + + # Initialize the metadata + meta = pysat.Meta() + for col in data.columns: + meta[col] = { + meta.labels.units: 'SFU', meta.labels.notes: '', + meta.labels.name: 'NoRP solar sadio flux {:} w/AU corr'.format( + col.replace("_", " ")), + meta.labels.desc: ''.join([ + 'NoRP solar radio flux at ', col.replace("_", " "), + ' with Astronomical Unit (AU) correction in Solar Flux Units', + ' (SFU).']), + meta.labels.fill_val: np.nan, + meta.labels.min_val: 0, meta.labels.max_val: np.inf} + + return data, meta + + +def list_files(tag='', inst_id='', data_path='', format_str=None): + """List local NoRP solar radio flux data files. + + Parameters + ---------- + tag : str + Instrument tag, accepts any value from `tags`. (default='') + inst_id : str + Instrument ID, not used. (default='') + data_path : str + Path to data directory. (default='') + format_str : str or NoneType + User specified file format. If None is specified, the default + formats associated with the supplied tags are used. (default=None) + + Returns + ------- + out_files : pds.Series + A Series containing the verified available files + + Note + ---- + Called by pysat. Not intended for direct use by user. + + """ + # Files are by month, going to add date to monthly filename for each day of + # the month. The load routine will load a month of data and use the + # appended date to select out appropriate data. + if format_str is None: + format_str = "_".join(["norp", "rf", tag, '{year:04d}-{month:02d}.txt']) + + out_files = pysat.Files.from_os(data_path=data_path, format_str=format_str) + if not out_files.empty: + out_files.loc[out_files.index[-1] + pds.DateOffset(months=1) + - pds.DateOffset(days=1)] = out_files.iloc[-1] + out_files = out_files.asfreq('D', 'pad') + out_files = out_files + '_' + out_files.index.strftime('%Y-%m-%d') + + return out_files + + +def download(date_array, tag, inst_id, data_path, update_files=False, + mock_download_dir=None): + """Download NoRP solar radio flux data. + + Parameters + ---------- + date_array : array-like + Sequence of dates for which files will be downloaded. + tag : str + Denotes type of file to load. + inst_id : str + Specifies the satellite ID for a constellation. + data_path : str + Path to data directory. + update_files : bool + Re-download data for files that already exist if True (default=False) + mock_download_dir : str or NoneType + Local directory with downloaded files or None. If not None, will + process any files with the correct name and date as if they were + downloaded (default=None) + + Raises + ------ + IOError + If a problem is encountered connecting to the gateway or retrieving + data from the remote or local repository. + + Note + ---- + Called by pysat. Not intended for direct use by user. + + """ + # Download the daily radio flux data from NoRP, saving data in monthly files + methods.norp.daily_rf_downloads(data_path, + mock_download_dir=mock_download_dir, + start=date_array[0], stop=date_array[-1]) + + return From 81134b9bc74c095b9a3b332927291ecd1baa9514 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Wed, 14 Aug 2024 16:44:11 -0400 Subject: [PATCH 4/6] DOC: updated supported instruments Updated the documentation to include the new Instrument. --- docs/supported_instruments.rst | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/docs/supported_instruments.rst b/docs/supported_instruments.rst index 04fc8e4..0e18db4 100644 --- a/docs/supported_instruments.rst +++ b/docs/supported_instruments.rst @@ -66,6 +66,30 @@ Supports ACE Solar Wind Electron Proton Alpha Monitor data. :members: +.. _norp-inst: +NoRP +---- + +The Nobeyama Radio Polarameters (NoRP) platform encompasses solar flux +measurements provided by the Japanese Solar Science Observatory. +`NoRP `_ provides +additional information and processing tools on their website. + +.. _norp-rf-inst: + +RF +^^^ + +RF is the radio flux measured from the sun at different wavelengths. This +provides a different measure of solar activity and has been corrected to be +in solar flux units at 1 AU. The daily data set currently starts in Nov 1951 +and is updated to extend to the current period, but not in real-time. + + +.. automodule:: pysatSpaceWeather.instruments.norp_rf + :members: + + .. _sw-inst: SW --- From 9b8b48bf27cb0dc893d209cc2c471aa298efdbd9 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Wed, 14 Aug 2024 16:44:45 -0400 Subject: [PATCH 5/6] TST: added unit tests for NoRP data Added unit tests for the new NoRP support functions and a data file to test mock downloads for the new Instrument. --- .../tests/test_data/TYKW-NoRP_dailyflux.txt | 7 +++ pysatSpaceWeather/tests/test_methods_norp.py | 55 +++++++++++++++++++ 2 files changed, 62 insertions(+) create mode 100644 pysatSpaceWeather/tests/test_data/TYKW-NoRP_dailyflux.txt create mode 100644 pysatSpaceWeather/tests/test_methods_norp.py diff --git a/pysatSpaceWeather/tests/test_data/TYKW-NoRP_dailyflux.txt b/pysatSpaceWeather/tests/test_data/TYKW-NoRP_dailyflux.txt new file mode 100644 index 0000000..789b026 --- /dev/null +++ b/pysatSpaceWeather/tests/test_data/TYKW-NoRP_dailyflux.txt @@ -0,0 +1,7 @@ +TYKW-NoRP solar daily flux (1951-11-02 -- 1951-11-06) +Date,1 GHz,2 GHz,3.75 GHz,9.4 GHz +"1951-11-02",NaN,NaN,NaN,NaN +"1951-11-03",NaN,NaN,NaN,NaN +"1951-11-04",NaN,NaN,NaN,NaN +"1951-11-05",NaN,NaN,NaN,NaN +"1951-11-06",NaN,NaN,115.000,NaN diff --git a/pysatSpaceWeather/tests/test_methods_norp.py b/pysatSpaceWeather/tests/test_methods_norp.py new file mode 100644 index 0000000..e042d04 --- /dev/null +++ b/pysatSpaceWeather/tests/test_methods_norp.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python +# Full license can be found in License.md +# Full author list can be found in .zenodo.json file +# DOI:10.5281/zenodo.3986138 +# ---------------------------------------------------------------------------- +"""Integration and unit test suite for NoRP methods.""" + +import pytest + +from pysatSpaceWeather.instruments.methods import norp as mm_norp + + +class TestNoRPMethods(object): + """Test class for NoRP methods.""" + + def setup_method(self): + """Create a clean testing setup.""" + self.out = None + return + + def teardown_method(self): + """Clean up previous testing setup.""" + del self.out + return + + def test_acknowledgements(self): + """Test the NoRP acknowledgements.""" + self.out = mm_norp.acknowledgements() + assert self.out.find('NoRP') >= 0 + return + + @pytest.mark.parametrize('name,tag', [('rf', 'daily')]) + def test_references(self, name, tag): + """Test the references for a NoRP instrument. + + Parameters + ---------- + name : str + Instrument name + tag : str + Instrument tag + + """ + self.out = mm_norp.references(name, tag) + assert self.out.find('Toyokawa Observatory') > 0 + return + + def test_references_bad_name(self): + """Test the references raise an informative error for bad instrument.""" + with pytest.raises(KeyError) as kerr: + mm_norp.references('ace', 'sis') + + assert str(kerr.value).find('ace') >= 0, \ + "Unknown KeyError message: {:}".format(kerr.value) + return From f2adc783dde8fcd4b7087acec458a8c2f6a97d97 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Wed, 14 Aug 2024 16:45:15 -0400 Subject: [PATCH 6/6] DOC: updated Changelog Bumped the next release version and added a summary of this pull request to the changes. --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f89ef1..2c920ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,8 +3,10 @@ Change Log All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](https://semver.org/). -[0.1.X] - 2024-XX-XX +[0.2.0] - 2024-XX-XX -------------------- +* Enhancements + * Added an instrument for the daily Nobeyama Radio Polarimeters solar flux * Maintenance * Removed unneeded keyword arguments from Kp method functions * Replaces `fillna` with `asfreq` to maintain the same behaviour