From 916dc297952d8226347ab09d0db0cab8d5551919 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Tue, 30 Jul 2024 17:15:34 -0400 Subject: [PATCH 01/30] BUG: avoid certificate experation Avoid certificate expiration by using the `verify` flag set to False. --- CHANGELOG.md | 1 + pysatSpaceWeather/instruments/methods/gfz.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b31c4d7..ac105f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). -------------------- * Maintenance * Removed unneeded keyword arguments from Kp method functions + * Added `verify=False` to GFZ requests [0.1.0] - 2024-02-16 -------------------- diff --git a/pysatSpaceWeather/instruments/methods/gfz.py b/pysatSpaceWeather/instruments/methods/gfz.py index 12ffceb..2d5ad55 100644 --- a/pysatSpaceWeather/instruments/methods/gfz.py +++ b/pysatSpaceWeather/instruments/methods/gfz.py @@ -128,7 +128,7 @@ def json_downloads(date_array, data_path, local_file_prefix, local_date_fmt, query_url = '{:s}&status=def'.format(query_url) # The data is returned as a JSON file - req = requests.get(query_url) + req = requests.get(query_url, verify=False) # Process the JSON file if req.text.find('Gateway Timeout') >= 0: @@ -249,7 +249,7 @@ def kp_ap_cp_download(platform, name, date_array, tag, inst_id, data_path, dl_date.strftime('%Y')))) if mock_download_dir is None: furl = ''.join([burl, fname]) - req = requests.get(furl) + req = requests.get(furl, verify=False) raw_txt = req.text if req.ok else None else: From f0995e2434eaafde3cc4ae074ccc1a50694d4ed2 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 12 Aug 2024 17:16:13 -0400 Subject: [PATCH 02/30] MAINT: updated pandas fill method in f107 Updated the pandas fill method, replacing the deprecated `fillna` with `ffill`. --- pysatSpaceWeather/instruments/methods/f107.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pysatSpaceWeather/instruments/methods/f107.py b/pysatSpaceWeather/instruments/methods/f107.py index 3fe7edf..48c8709 100644 --- a/pysatSpaceWeather/instruments/methods/f107.py +++ b/pysatSpaceWeather/instruments/methods/f107.py @@ -304,7 +304,7 @@ def combine_f107(standard_inst, forecast_inst, start=None, stop=None): # Resample the output data, filling missing values if (date_range.shape != f107_inst.index.shape or abs(date_range - f107_inst.index).max().total_seconds() > 0.0): - f107_inst.data = f107_inst.data.resample(freq).fillna(method=None) + f107_inst.data = f107_inst.data.resample(freq).ffill() if np.isfinite(fill_val): f107_inst.data[np.isnan(f107_inst.data)] = fill_val @@ -354,7 +354,7 @@ def calc_f107a(f107_inst, f107_name='f107', f107a_name='f107a', min_pnts=41): # # Ensure the data are evenly sampled at a daily frequency, since this is # how often F10.7 is calculated. - f107_fill = f107_inst.data.resample('1D').fillna(method=None) + f107_fill = f107_inst.data.resample('1D').ffill() # Replace the time index with an ordinal time_ind = f107_fill.index From bd271fee7ccda87236661a946c813042214ae442 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 12 Aug 2024 17:16:39 -0400 Subject: [PATCH 03/30] MAINT: updated pandas in kp_ap Updated the pandas indexing and fill method, replacing the deprecated `fillna` with `ffill`. --- pysatSpaceWeather/instruments/methods/kp_ap.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/pysatSpaceWeather/instruments/methods/kp_ap.py b/pysatSpaceWeather/instruments/methods/kp_ap.py index efc2eec..d33aa0a 100644 --- a/pysatSpaceWeather/instruments/methods/kp_ap.py +++ b/pysatSpaceWeather/instruments/methods/kp_ap.py @@ -306,9 +306,9 @@ def calc_daily_Ap(ap_inst, ap_name='3hr_ap', daily_name='Ap', index=[ap_mean.index[0] - pds.DateOffset(hours=3)]) # Extract the mean that only uses data for one day - ap_sel = ap_pad.combine_first(ap_mean[[i for i, tt in - enumerate(ap_mean.index) - if tt.hour == 21]]) + ap_sel = ap_pad.combine_first(ap_mean.iloc[[i for i, tt in + enumerate(ap_mean.index) + if tt.hour == 21]]) # Backfill this data ap_data = ap_sel.resample('3H').bfill() @@ -665,9 +665,12 @@ def combine_kp(standard_inst=None, recent_inst=None, forecast_inst=None, good_vals = recent_inst['Kp'][good_times] != local_fill_val # Save output data and cycle time - kp_times.extend(list(recent_inst.index[good_times][good_vals])) - kp_values.extend(list(recent_inst['Kp'][good_times][good_vals])) - itime = kp_times[-1] + pds.DateOffset(hours=3) + if len(good_vals): + kp_times.extend(list( + recent_inst.index[good_times][good_vals])) + kp_values.extend(list( + recent_inst['Kp'][good_times][good_vals])) + itime = kp_times[-1] + pds.DateOffset(hours=3) inst_flag = 'forecast' if forecast_inst is not None else None notes += "{:})".format(itime.date()) @@ -751,7 +754,7 @@ def combine_kp(standard_inst=None, recent_inst=None, forecast_inst=None, # Resample the output data, filling missing values if (date_range.shape != kp_inst.index.shape or abs(date_range - kp_inst.index).max().total_seconds() > 0.0): - kp_inst.data = kp_inst.data.resample(freq).fillna(method=None) + kp_inst.data = kp_inst.data.resample(freq).ffill() if np.isfinite(fill_val): kp_inst.data[np.isnan(kp_inst.data)] = fill_val From d86c91a2f42d7807ebee1454bb96e49df139206e Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 12 Aug 2024 17:17:06 -0400 Subject: [PATCH 04/30] STY: added logger output Added logger output for file creation at the info level. --- pysatSpaceWeather/instruments/methods/swpc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pysatSpaceWeather/instruments/methods/swpc.py b/pysatSpaceWeather/instruments/methods/swpc.py index 0635a59..e4965c9 100644 --- a/pysatSpaceWeather/instruments/methods/swpc.py +++ b/pysatSpaceWeather/instruments/methods/swpc.py @@ -318,6 +318,7 @@ def rewrite_daily_solar_data_file(year, outfiles, lines): # Write out as a file data.to_csv(outfiles[data_name], header=True) + pysat.logger.info('Wrote: {:}'.format(outfiles[data_name])) return From 068da4bd2f11d6db26e9af2f067c5f39efd7860d Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 12 Aug 2024 17:17:41 -0400 Subject: [PATCH 05/30] MAINT: updated pandas in `sw_f107` Updated the pandas series indexing method. --- pysatSpaceWeather/instruments/sw_f107.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatSpaceWeather/instruments/sw_f107.py b/pysatSpaceWeather/instruments/sw_f107.py index d3ca6b5..0bcf106 100644 --- a/pysatSpaceWeather/instruments/sw_f107.py +++ b/pysatSpaceWeather/instruments/sw_f107.py @@ -399,7 +399,7 @@ def download(date_array, tag, inst_id, data_path, update_files=False, # Cut the date from the end of the local files for i, lfile in enumerate(local_files): - local_files[i] = lfile[:-11] + local_files.iloc[i] = lfile[:-11] methods.swpc.old_indices_dsd_download( name, date_array, data_path, local_files, today, From 4b9c8c88158ad2db4f19995055fcdc3230217f62 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 12 Aug 2024 17:26:03 -0400 Subject: [PATCH 06/30] DOC: updated NCEI name Updated the NCEI name. Did not add a link, as their website is not stable. --- docs/supported_instruments.rst | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/supported_instruments.rst b/docs/supported_instruments.rst index efb508a..04fc8e4 100644 --- a/docs/supported_instruments.rst +++ b/docs/supported_instruments.rst @@ -179,10 +179,9 @@ Dst ^^^ The Disturbance Storm Time (Dst) Index is a measure of magnetic activity -associated with the ring current. The National Geophysical Data Center (NGDC) -maintains the -`current database `_ from which -the historic Dst is downloaded. +associated with the ring current. The National Centers for Environmental +Information (NCEI), formerly the National Geophysical Data Center (NGDC), +maintains the current database from which the historic Dst is downloaded. `LASP `_ performs the calculates and provides the predicted Dst for the last 96 hours. You can learn more about the Dst Index at the From 445f7f8e38e4d8c1d7ae60620f04f499ec776981 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 12 Aug 2024 17:26:35 -0400 Subject: [PATCH 07/30] DOC: added links to ignore Added links to ignore in the unit tests, as they are file but require certificates to access. --- docs/conf.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index e35e7b2..637b683 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -181,4 +181,5 @@ # Links to ignore when checking for stability linkcheck_ignore = ['https://lasp.colorado.edu/space_weather/dsttemerin/', - 'https://*QUERY'] + 'https://*QUERY', + 'https://datapub.gfz-potsdam.de/download/10.5880.Kp.0001/*'] From 6d375fea9823dabe2034e2c388061c70ee5b8ba7 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 12 Aug 2024 17:28:35 -0400 Subject: [PATCH 08/30] DOC: update summary of changes Updated changelog. --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac105f3..d291677 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,10 @@ This project adheres to [Semantic Versioning](https://semver.org/). -------------------- * Maintenance * Removed unneeded keyword arguments from Kp method functions + * Replaces `fillna` with `ffill` + * Implemented `iloc` in pandas Series and DataFrame index access * Added `verify=False` to GFZ requests + * Updated documentation links and fixed intersphinx mapping [0.1.0] - 2024-02-16 -------------------- From d042597c248db6d472cbde766b604bd6458ad819 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 12 Aug 2024 17:28:51 -0400 Subject: [PATCH 09/30] MAINT: updated intersphinx mapping Updated intersphinx mapping. --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 637b683..a9ca856 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -177,7 +177,7 @@ epub_exclude_files = ['search.html'] # Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {'https://docs.python.org/': None} +intersphinx_mapping = {'': ('https://docs.python.org/': None)} # Links to ignore when checking for stability linkcheck_ignore = ['https://lasp.colorado.edu/space_weather/dsttemerin/', From e1b5e25a94c2fb70396ad46937336a0e291bfa6a Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 12 Aug 2024 17:39:26 -0400 Subject: [PATCH 10/30] MAINT: cycled python versions Removed Python 3.9 from listed support. --- pyproject.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index fe8ae65..1df1f05 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,6 @@ classifiers = [ "Natural Language :: English", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", @@ -50,7 +49,6 @@ dependencies = [ [project.optional-dependencies] test = [ - "coveralls < 3.3", "flake8", "flake8-docstrings", "hacking >= 1.0", From db1932996a6287c7437fc536bbaf6c313f758f74 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 12 Aug 2024 17:40:12 -0400 Subject: [PATCH 11/30] MAINT: updated main CI test Updated the main CI test by: - cycling the python versions tested, - updated NEP29, and - attempting to use the coveralls action. --- .github/workflows/main.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d6efa7d..e7da6f2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,13 +16,13 @@ jobs: fail-fast: false matrix: os: ["ubuntu-latest", "macos-latest", "windows-latest"] - python-version: ["3.10", "3.11", "3.12"] + python-version: ["3.11", "3.12"] numpy_ver: ["latest"] test_config: ["latest"] include: # NEP29 compliance settings - - python-version: "3.9" - numpy_ver: "1.23" + - python-version: "3.10" + numpy_ver: "1.24" os: ubuntu-latest test_config: "NEP29" # Operational compliance settings @@ -72,11 +72,13 @@ jobs: - name: Test with pytest run: pytest - - name: Publish results to coveralls - env: - GITHUB_TOKEN: ${{ secrets.github_token }} - COVERALLS_PARALLEL: true - run: coveralls --rcfile=pyproject.toml --service=github + - name: Coveralls Parallel + uses: coverallsapp/github-actions@v2.3.0 + with: + flag-name: run=${{ join(matrix.*, '-') }} + parallel: true + format: cobertura + debug: true finish: name: Finish Coverage Analysis @@ -84,8 +86,6 @@ jobs: runs-on: ubuntu-latest steps: - name: Coveralls Finished - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - pip install --upgrade coveralls - coveralls --service=github --finish + uses: coverallsapp/github-action@v2.3.0 + with: + parallel-finished: true From 688135fc881bb88a932b3d3eedec4216ff09acdd Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 12 Aug 2024 17:41:50 -0400 Subject: [PATCH 12/30] BUG: fixed syntax Fixed the syntax for the tuple in the intersphinx mapping. --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index a9ca856..acf098b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -177,7 +177,7 @@ epub_exclude_files = ['search.html'] # Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {'': ('https://docs.python.org/': None)} +intersphinx_mapping = {'': ('https://docs.python.org/', None)} # Links to ignore when checking for stability linkcheck_ignore = ['https://lasp.colorado.edu/space_weather/dsttemerin/', From 756a061a0645974f468c290f24179fe1d9a7a679 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 12 Aug 2024 17:45:54 -0400 Subject: [PATCH 13/30] MAINT: cycle python to highest supported version Cycle the pysat_rc tests to use the highest supported version. --- .github/workflows/pysat_rc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pysat_rc.yml b/.github/workflows/pysat_rc.yml index 3fe807c..9d4f419 100644 --- a/.github/workflows/pysat_rc.yml +++ b/.github/workflows/pysat_rc.yml @@ -12,7 +12,7 @@ jobs: fail-fast: false matrix: os: ["ubuntu-latest", "macos-latest", "windows-latest"] - python-version: ["3.11"] + python-version: ["3.12"] name: Python ${{ matrix.python-version }} on ${{ matrix.os }} runs-on: ${{ matrix.os }} From ee310e0ae69fb6791760b9b38479353605623f88 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 12 Aug 2024 17:47:03 -0400 Subject: [PATCH 14/30] BUG: replaced tab with spaces Fixed yaml syntax. --- .github/workflows/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e7da6f2..85276b2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -76,9 +76,9 @@ jobs: uses: coverallsapp/github-actions@v2.3.0 with: flag-name: run=${{ join(matrix.*, '-') }} - parallel: true - format: cobertura - debug: true + parallel: true + format: cobertura + debug: true finish: name: Finish Coverage Analysis From c6cce4f8582c9b69bda4c09bfb87fe3051c7718d Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Tue, 13 Aug 2024 08:47:26 -0400 Subject: [PATCH 15/30] BUG: added token to coveralls action Added the github token secret specification to the coveralls action. --- .github/workflows/main.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 85276b2..86d6b21 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -73,8 +73,9 @@ jobs: run: pytest - name: Coveralls Parallel - uses: coverallsapp/github-actions@v2.3.0 + uses: coverallsapp/github-actions@v2 with: + github-token: ${{ secrets.GITHUB_TOKEN }} flag-name: run=${{ join(matrix.*, '-') }} parallel: true format: cobertura @@ -86,6 +87,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Coveralls Finished - uses: coverallsapp/github-action@v2.3.0 + uses: coverallsapp/github-action@v2 with: + github-token: ${{ secrets.GITHUB_TOKEN }} parallel-finished: true From f9353d8ff749e3903ca57807d0c39351d4e4919b Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Tue, 13 Aug 2024 08:49:16 -0400 Subject: [PATCH 16/30] BUG: removed typo Removed 's' in call to GitHub Actions. --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 86d6b21..6558eca 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -73,7 +73,7 @@ jobs: run: pytest - name: Coveralls Parallel - uses: coverallsapp/github-actions@v2 + uses: coverallsapp/github-action@v2 with: github-token: ${{ secrets.GITHUB_TOKEN }} flag-name: run=${{ join(matrix.*, '-') }} From 8963895a7513dfb1d6c7ecb104164ce49560a008 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Tue, 13 Aug 2024 10:46:46 -0400 Subject: [PATCH 17/30] BUG: change `ffill` to `asfreq` The correct change for resampling to maintain the old behaviour was to change `fillna` with `asfreq`. --- CHANGELOG.md | 2 +- pysatSpaceWeather/instruments/methods/f107.py | 6 +++--- pysatSpaceWeather/instruments/methods/kp_ap.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d291677..2f89ef1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). -------------------- * Maintenance * Removed unneeded keyword arguments from Kp method functions - * Replaces `fillna` with `ffill` + * Replaces `fillna` with `asfreq` to maintain the same behaviour * Implemented `iloc` in pandas Series and DataFrame index access * Added `verify=False` to GFZ requests * Updated documentation links and fixed intersphinx mapping diff --git a/pysatSpaceWeather/instruments/methods/f107.py b/pysatSpaceWeather/instruments/methods/f107.py index 48c8709..9a792e3 100644 --- a/pysatSpaceWeather/instruments/methods/f107.py +++ b/pysatSpaceWeather/instruments/methods/f107.py @@ -304,7 +304,7 @@ def combine_f107(standard_inst, forecast_inst, start=None, stop=None): # Resample the output data, filling missing values if (date_range.shape != f107_inst.index.shape or abs(date_range - f107_inst.index).max().total_seconds() > 0.0): - f107_inst.data = f107_inst.data.resample(freq).ffill() + f107_inst.data = f107_inst.data.resample(freq).asfreq() if np.isfinite(fill_val): f107_inst.data[np.isnan(f107_inst.data)] = fill_val @@ -354,7 +354,7 @@ def calc_f107a(f107_inst, f107_name='f107', f107a_name='f107a', min_pnts=41): # # Ensure the data are evenly sampled at a daily frequency, since this is # how often F10.7 is calculated. - f107_fill = f107_inst.data.resample('1D').ffill() + f107_fill = f107_inst.data.resample('1D').asfreq() # Replace the time index with an ordinal time_ind = f107_fill.index @@ -375,7 +375,7 @@ def calc_f107a(f107_inst, f107_name='f107', f107a_name='f107a', min_pnts=41): freq = pysat.utils.time.calc_freq(f107_inst.index) if freq != "86400S": # Resample to the desired frequency - f107_fill = f107_fill.resample(freq).ffill() + f107_fill = f107_fill.resample(freq).asfreq() # Save the output in a list f107a = list(f107_fill[f107a_name]) diff --git a/pysatSpaceWeather/instruments/methods/kp_ap.py b/pysatSpaceWeather/instruments/methods/kp_ap.py index d33aa0a..35756b6 100644 --- a/pysatSpaceWeather/instruments/methods/kp_ap.py +++ b/pysatSpaceWeather/instruments/methods/kp_ap.py @@ -754,7 +754,7 @@ def combine_kp(standard_inst=None, recent_inst=None, forecast_inst=None, # Resample the output data, filling missing values if (date_range.shape != kp_inst.index.shape or abs(date_range - kp_inst.index).max().total_seconds() > 0.0): - kp_inst.data = kp_inst.data.resample(freq).ffill() + kp_inst.data = kp_inst.data.resample(freq).asfreq() if np.isfinite(fill_val): kp_inst.data[np.isnan(kp_inst.data)] = fill_val From 61c5964e0f20932a19fcd733d21e89cdd4ccda6c Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Tue, 13 Aug 2024 11:00:48 -0400 Subject: [PATCH 18/30] BUG: fill for sub-daily frequencies If the frequency is less than a day, use the mean for the entire day. Otherwise, include a fill value. --- pysatSpaceWeather/instruments/methods/f107.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pysatSpaceWeather/instruments/methods/f107.py b/pysatSpaceWeather/instruments/methods/f107.py index 9a792e3..6156fae 100644 --- a/pysatSpaceWeather/instruments/methods/f107.py +++ b/pysatSpaceWeather/instruments/methods/f107.py @@ -375,12 +375,15 @@ def calc_f107a(f107_inst, f107_name='f107', f107a_name='f107a', min_pnts=41): freq = pysat.utils.time.calc_freq(f107_inst.index) if freq != "86400S": # Resample to the desired frequency - f107_fill = f107_fill.resample(freq).asfreq() + if pds.to_timedelta(freq) < pds.to_timedelta("86400S"): + f107_fill = f107_fill.resample(freq).ffill() + else: + f107_fill = f107_fill.resample(freq).asfreq() # Save the output in a list f107a = list(f107_fill[f107a_name]) - # Fill any dates that fall + # Fill any dates that fall just outside of the range time_ind = pds.date_range(f107_fill.index[0], f107_inst.index[-1], freq=freq) for itime in time_ind[f107_fill.index.shape[0]:]: From a06ebbaa2ac040f65f8a349ff12274675f0cc526 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Tue, 13 Aug 2024 11:01:18 -0400 Subject: [PATCH 19/30] TST: allow local testing `addopts` was interfering with local testing, move back to test yaml. --- .github/workflows/main.yml | 2 +- pyproject.toml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6558eca..5a3e99d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -70,7 +70,7 @@ jobs: run: flake8 . --count --exit-zero --max-complexity=10 --statistics - name: Test with pytest - run: pytest + run: pytest --cov=pysatSpaceWeather - name: Coveralls Parallel uses: coverallsapp/github-action@v2 diff --git a/pyproject.toml b/pyproject.toml index 1df1f05..2c14319 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,7 +73,6 @@ Source = "https://github.com/pysat/pysatSpaceWeather" [tool.coverage.report] [tool.pytest.ini_options] -addopts = "--cov=pysatSpaceWeather" markers = [ "all_inst", "download", From 1bf57750275230074ff839cc6d0020c87ccd9792 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Tue, 13 Aug 2024 11:05:34 -0400 Subject: [PATCH 20/30] BUG: include coveralls in test install Probably need coveralls installed to get something to report. --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 2c14319..d024552 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,6 +49,7 @@ dependencies = [ [project.optional-dependencies] test = [ + "coveralls", "flake8", "flake8-docstrings", "hacking >= 1.0", From c13247cfb63d99bc92d2872d3c0aa932d52871af Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Tue, 13 Aug 2024 11:22:51 -0400 Subject: [PATCH 21/30] MAINT: updated pandas frequency Update the format of the pandas frequency to address FutureWarnings. --- pysatSpaceWeather/instruments/methods/f107.py | 4 ++-- pysatSpaceWeather/instruments/methods/kp_ap.py | 2 +- pysatSpaceWeather/instruments/methods/swpc.py | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pysatSpaceWeather/instruments/methods/f107.py b/pysatSpaceWeather/instruments/methods/f107.py index 6156fae..aa40565 100644 --- a/pysatSpaceWeather/instruments/methods/f107.py +++ b/pysatSpaceWeather/instruments/methods/f107.py @@ -373,9 +373,9 @@ def calc_f107a(f107_inst, f107_name='f107', f107a_name='f107a', min_pnts=41): # Resample to the original frequency, if it is not equal to 1 day freq = pysat.utils.time.calc_freq(f107_inst.index) - if freq != "86400S": + if freq != "86400s": # Resample to the desired frequency - if pds.to_timedelta(freq) < pds.to_timedelta("86400S"): + if pds.to_timedelta(freq) < pds.to_timedelta("86400s"): f107_fill = f107_fill.resample(freq).ffill() else: f107_fill = f107_fill.resample(freq).asfreq() diff --git a/pysatSpaceWeather/instruments/methods/kp_ap.py b/pysatSpaceWeather/instruments/methods/kp_ap.py index 35756b6..4dfeb4b 100644 --- a/pysatSpaceWeather/instruments/methods/kp_ap.py +++ b/pysatSpaceWeather/instruments/methods/kp_ap.py @@ -311,7 +311,7 @@ def calc_daily_Ap(ap_inst, ap_name='3hr_ap', daily_name='Ap', if tt.hour == 21]]) # Backfill this data - ap_data = ap_sel.resample('3H').bfill() + ap_data = ap_sel.resample('3h').bfill() # Save the output for the original time range ap_inst[daily_name] = pds.Series(ap_data[1:], index=ap_data.index[1:]) diff --git a/pysatSpaceWeather/instruments/methods/swpc.py b/pysatSpaceWeather/instruments/methods/swpc.py index e4965c9..0d0ffc3 100644 --- a/pysatSpaceWeather/instruments/methods/swpc.py +++ b/pysatSpaceWeather/instruments/methods/swpc.py @@ -479,7 +479,7 @@ def solar_geomag_predictions_download(name, date_array, data_path, # Process the Kp data hr_strs = ['00-03UT', '03-06UT', '06-09UT', '09-12UT', '12-15UT', '15-18UT', '18-21UT', '21-00UT'] - data_times['kp'] = pds.date_range(pred_times[0], periods=24, freq='3H') + data_times['kp'] = pds.date_range(pred_times[0], periods=24, freq='3h') for line in kp_raw.split('\n'): if line.find("Prob_Mid") >= 0: @@ -644,7 +644,7 @@ def geomag_forecast_download(name, date_array, data_path, kp_day2.append(float(cols[-2])) kp_day3.append(float(cols[-1])) - kp_times = pds.date_range(forecast_date, periods=24, freq='3H') + kp_times = pds.date_range(forecast_date, periods=24, freq='3h') kp_day = [] for dd in [kp_day1, kp_day2, kp_day3]: kp_day.extend(dd) @@ -785,7 +785,7 @@ def kp_ap_recent_download(name, date_array, data_path, mock_download_dir=None): sub_aps[i].append(np.int64(ap_sub_lines[i])) # Create times on 3 hour cadence - kp_times = pds.date_range(times[0], periods=(8 * 30), freq='3H') + kp_times = pds.date_range(times[0], periods=(8 * 30), freq='3h') # Put both data sets into DataFrames data = {'kp': pds.DataFrame({'mid_lat_Kp': sub_kps[0], From 06369e65cd9393b35670539e00587c1bbb7307dc Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Tue, 13 Aug 2024 11:23:11 -0400 Subject: [PATCH 22/30] MAINT: implement pandas iloc Implement pandas iloc to address FutureWarnings. --- pysatSpaceWeather/instruments/ace_sis.py | 6 ++++-- pysatSpaceWeather/tests/test_methods_kp.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pysatSpaceWeather/instruments/ace_sis.py b/pysatSpaceWeather/instruments/ace_sis.py index 61f6546..5ad0596 100644 --- a/pysatSpaceWeather/instruments/ace_sis.py +++ b/pysatSpaceWeather/instruments/ace_sis.py @@ -115,8 +115,10 @@ def clean(self): # Evaluate the different proton fluxes. Replace bad values with NaN and # times with no valid data - self.data['int_pflux_10MeV'][self.data['status_10'] > max_status] = np.nan - self.data['int_pflux_30MeV'][self.data['status_30'] > max_status] = np.nan + self.data['int_pflux_10MeV'] = self.data['int_pflux_10MeV'].where( + (self.data['status_10'] <= max_status), other=np.nan) + self.data['int_pflux_30MeV'] = self.data['int_pflux_30MeV'].where( + (self.data['status_30'] <= max_status), other=np.nan) eval_cols = ['int_pflux_10MeV', 'int_pflux_30MeV'] diff --git a/pysatSpaceWeather/tests/test_methods_kp.py b/pysatSpaceWeather/tests/test_methods_kp.py index 9f10e3c..e6e830a 100644 --- a/pysatSpaceWeather/tests/test_methods_kp.py +++ b/pysatSpaceWeather/tests/test_methods_kp.py @@ -395,7 +395,7 @@ def test_convert_ap_to_kp_middle(self): """Test conversion of ap to Kp where ap is not an exact Kp value.""" kp_ap.convert_3hr_kp_to_ap(self.testInst) - new_val = self.testInst['3hr_ap'][8] + 1 + new_val = self.testInst['3hr_ap']iloc[8] + 1 self.testInst.data.at[self.testInst.index[8], '3hr_ap'] = new_val kp_out, kp_meta = kp_ap.convert_ap_to_kp(self.testInst['3hr_ap']) From c2f9f325cbdefd1d65b34679b3d64f2ce499b3f9 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Tue, 13 Aug 2024 11:28:40 -0400 Subject: [PATCH 23/30] TST: ensure output is xml Ensure coveralls output is xml. --- .github/workflows/main.yml | 2 +- pyproject.toml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5a3e99d..83b3355 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -70,7 +70,7 @@ jobs: run: flake8 . --count --exit-zero --max-complexity=10 --statistics - name: Test with pytest - run: pytest --cov=pysatSpaceWeather + run: pytest --cov=pysatSpaceWeather --cov-report xml - name: Coveralls Parallel uses: coverallsapp/github-action@v2 diff --git a/pyproject.toml b/pyproject.toml index d024552..2c14319 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,7 +49,6 @@ dependencies = [ [project.optional-dependencies] test = [ - "coveralls", "flake8", "flake8-docstrings", "hacking >= 1.0", From 2c5a0ec3b34199dbe167d4ab2057bab57905a312 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Tue, 13 Aug 2024 11:29:42 -0400 Subject: [PATCH 24/30] BUG: fixed typo Fixed a typo in the new `iloc` call. --- pysatSpaceWeather/tests/test_methods_kp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatSpaceWeather/tests/test_methods_kp.py b/pysatSpaceWeather/tests/test_methods_kp.py index e6e830a..8e6c80e 100644 --- a/pysatSpaceWeather/tests/test_methods_kp.py +++ b/pysatSpaceWeather/tests/test_methods_kp.py @@ -395,7 +395,7 @@ def test_convert_ap_to_kp_middle(self): """Test conversion of ap to Kp where ap is not an exact Kp value.""" kp_ap.convert_3hr_kp_to_ap(self.testInst) - new_val = self.testInst['3hr_ap']iloc[8] + 1 + new_val = self.testInst['3hr_ap'].iloc[8] + 1 self.testInst.data.at[self.testInst.index[8], '3hr_ap'] = new_val kp_out, kp_meta = kp_ap.convert_ap_to_kp(self.testInst['3hr_ap']) From 0be7074f8fc280f10ec362111d7c7f7a00b4b1a8 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Tue, 13 Aug 2024 11:37:22 -0400 Subject: [PATCH 25/30] TST: force coveralls to finish Try and force coveralls to finish even if some jobs fail. --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 83b3355..c1a9c14 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -91,3 +91,4 @@ jobs: with: github-token: ${{ secrets.GITHUB_TOKEN }} parallel-finished: true + fail-on-error: false From 5d5eaa0b41c7e4edeb5d7c4558d2342319cea09c Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Tue, 13 Aug 2024 11:47:14 -0400 Subject: [PATCH 26/30] TST: try carryforward Try carryforward instead of 'fail-on-error'. --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c1a9c14..77616c1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -91,4 +91,4 @@ jobs: with: github-token: ${{ secrets.GITHUB_TOKEN }} parallel-finished: true - fail-on-error: false + carryforward: "ubuntu-latest-3.12-latest-latest,ubuntu-latest-3.11-latest-latest,macos-latest-3.12-latest-latest,macos-latest-3.11-latest-latest,windows-latest-3.12-latest-latest,windows-latest-3.11-latest-latest,3.10-1.24-ubuntu-latest-NEP29" From 715f315071842c2a2b30fe7bc9b6dad7ff041715 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Tue, 13 Aug 2024 11:55:48 -0400 Subject: [PATCH 27/30] REV: changes were unnecessary Changes were unnecessary, lost track of maintenance updates. --- pysatSpaceWeather/instruments/methods/f107.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pysatSpaceWeather/instruments/methods/f107.py b/pysatSpaceWeather/instruments/methods/f107.py index aa40565..2f3b4a6 100644 --- a/pysatSpaceWeather/instruments/methods/f107.py +++ b/pysatSpaceWeather/instruments/methods/f107.py @@ -375,10 +375,7 @@ def calc_f107a(f107_inst, f107_name='f107', f107a_name='f107a', min_pnts=41): freq = pysat.utils.time.calc_freq(f107_inst.index) if freq != "86400s": # Resample to the desired frequency - if pds.to_timedelta(freq) < pds.to_timedelta("86400s"): - f107_fill = f107_fill.resample(freq).ffill() - else: - f107_fill = f107_fill.resample(freq).asfreq() + f107_fill = f107_fill.resample(freq).ffill() # Save the output in a list f107a = list(f107_fill[f107a_name]) From 081d8fc17927f17a635ca3516f53c985c8b5e0a8 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Tue, 13 Aug 2024 15:13:23 -0400 Subject: [PATCH 28/30] TST: added always tag Added the "always" yaml call to coveralls. --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 77616c1..eef233e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -84,6 +84,7 @@ jobs: finish: name: Finish Coverage Analysis needs: build + if: ${{ always() }} runs-on: ubuntu-latest steps: - name: Coveralls Finished From 217093b51cd796eea48fefc3e47c295cd175e843 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Tue, 13 Aug 2024 15:19:11 -0400 Subject: [PATCH 29/30] TST: remove carryforward Test removing the carryforward command. --- .github/workflows/main.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index eef233e..e93de2b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -92,4 +92,3 @@ jobs: with: github-token: ${{ secrets.GITHUB_TOKEN }} parallel-finished: true - carryforward: "ubuntu-latest-3.12-latest-latest,ubuntu-latest-3.11-latest-latest,macos-latest-3.12-latest-latest,macos-latest-3.11-latest-latest,windows-latest-3.12-latest-latest,windows-latest-3.11-latest-latest,3.10-1.24-ubuntu-latest-NEP29" From 73acafb42a61b8915e0aa690752ca5a881d55c16 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Tue, 13 Aug 2024 19:55:11 -0400 Subject: [PATCH 30/30] TST: update coveralls Update coveralls in pysat_rc tests. --- .github/workflows/pysat_rc.yml | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/.github/workflows/pysat_rc.yml b/.github/workflows/pysat_rc.yml index 9d4f419..a9afe6f 100644 --- a/.github/workflows/pysat_rc.yml +++ b/.github/workflows/pysat_rc.yml @@ -35,22 +35,25 @@ jobs: python -c "import pysat; pysat.params['data_dirs'] = 'pysatData'" - name: Test with pytest - run: pytest + run: pytest --cov=pysatSpaceWeather --cov-report xml - - name: Publish results to coveralls - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - COVERALLS_PARALLEL: true - run: coveralls --rcfile=pyproject.toml --service=github + - name: Coveralls Parallel + uses: coverallsapp/github-action@v2 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + flag-name: run=${{ join(matrix.*, '-') }} + parallel: true + format: cobertura + debug: true finish: name: Finish Coverage Analysis needs: build + if: ${{ always() }} runs-on: ubuntu-latest steps: - name: Coveralls Finished - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - pip install --upgrade coveralls - coveralls --service=github --finish + uses: coverallsapp/github-action@v2 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + parallel-finished: true