filter feature data for cloud flag if model isnt trained on this - in…

… the case of cloud type prediction model
NREL · Aug 26, 2024 · 21c5b6e · 21c5b6e
1 parent d856fb4
commit 21c5b6e
Show file tree

Hide file tree

Showing 3 changed files with 12 additions and 6 deletions.
diff --git a/nsrdb/gap_fill/mlclouds_fill.py b/nsrdb/gap_fill/mlclouds_fill.py
@@ -605,7 +605,10 @@ def predict_cld_properties(
                 col_slice
             )
         )
-        predict_feats = feature_df[self.phygnn_model.feature_names]
+        predict_feats = set(feature_df.columns).intersection(
+            self.phygnn_model.input_feature_names
+        )
+        predict_feats = feature_df[list(predict_feats)]
         if not low_mem:
             labels = self.phygnn_model.predict(predict_feats, table=False)
         else:

diff --git a/nsrdb/nsrdb.py b/nsrdb/nsrdb.py
@@ -762,10 +762,10 @@ def collect_data_model(
     @classmethod
     def collect_final(
         cls,
-        collect_dir,
         out_dir,
         year,
         grid,
+        collect_dir=None,
         freq='5min',
         var_meta=None,
         i_fname=None,
@@ -777,15 +777,15 @@ def collect_final(
 
         Parameters
         ----------
-        collect_dir : str
-            Directory with chunked files to be collected.
         out_dir : str
             Project directory.
         year : int | str
             Year of analysis
         grid : str
             Final/full NSRDB grid file. The first column must be the NSRDB
             site gid's.
+        collect_dir : str
+            Directory with chunked files to be collected.
         freq : str
             Final desired NSRDB temporal frequency.
         var_meta : str | pd.DataFrame | None
@@ -804,7 +804,7 @@ def collect_final(
             Logging level (DEBUG, INFO). If None, no logging will be
             initialized.
         """
-
+        collect_dir = collect_dir or os.path.join(out_dir, 'collect')
         nsrdb = cls(out_dir, year, grid, freq=freq, var_meta=var_meta)
         nsrdb._init_loggers(log_file=log_file, log_level=log_level)
         ti = nsrdb._parse_data_model_output_ti(nsrdb._daily_dir, freq)

diff --git a/nsrdb/utilities/sky_class.py b/nsrdb/utilities/sky_class.py
@@ -302,7 +302,10 @@ def add_validation_data(self, df):
         """Add NSRDB and SURFRAD ghi and dni data to a DataFrame."""
         df = df.reindex(self.nsrdb_time_index)
         assert len(df) == len(self.nsrdb_time_index)
-        ti_deltas = self.nsrdb_time_index - np.roll(self.nsrdb_time_index, 1)
+        ti_deltas = (
+            self.nsrdb_time_index.values[1:]
+            - self.nsrdb_time_index.values[:-1]
+        )
         ti_deltas_minutes = pd.Series(ti_deltas).dt.seconds / 60
         ti_delta_minutes = int(mode(ti_deltas_minutes)[0])
         freq = '{}T'.format(ti_delta_minutes)