diff --git a/sup3r/qa/qa.py b/sup3r/qa/qa.py index 10b6d712f..6a3a1a874 100644 --- a/sup3r/qa/qa.py +++ b/sup3r/qa/qa.py @@ -49,6 +49,7 @@ def __init__( t_enhance, temporal_coarsening_method, features=None, + source_features=None, output_names=None, input_handler_name=None, input_handler_kwargs=None, @@ -85,6 +86,17 @@ def __init__( Explicit list of features to validate. Can be a single feature str, list of string feature names, or None for all features found in the out_file_path. + source_features : str | list | None + Optional feature names to retrieve from the source dataset if the + source feature names are not the same as the sup3r output feature + names. These will be used to derive the features to be validated. + e.g. If model output is temperature_2m, and these were derived from + temperature_min_2m (and max), then source features should be + temperature_min_2m and temperature_max_2m while the model output + temperature_2m is aggregated using min/max in the + temporal_coarsening_method. Another example is features="ghi", + source_features="rsds", where this is a simple alternative name + lookup. output_names : str | list Optional output file dataset names corresponding to the features list input @@ -125,6 +137,11 @@ class for argument details. self._features = ( features if isinstance(features, (list, tuple)) else [features] ) + self._source_features = ( + source_features + if isinstance(source_features, (list, tuple)) + else [source_features] + ) self._out_names = ( output_names if isinstance(output_names, (list, tuple)) @@ -198,6 +215,14 @@ def output_names(self): return self.features return self._out_names + @property + def source_features(self): + """Get a list of source dataset names corresponding to the input source + data """ + if self._source_features is None or self._source_features == [None]: + return self.features + return self._source_features + @property def output_type(self): """Get output data type @@ -451,16 +476,17 @@ def run(self): """ errors = {} - ziter = zip(self.features, self.output_names) - for idf, (feature, dset_out) in enumerate(ziter): + ziter = zip(self.features, self.source_features, self.output_names) + for idf, (feature, source_feature, dset_out) in enumerate(ziter): logger.info( - 'Running QA on dataset {} of {} for "{}"'.format( - idf + 1, len(self.features), feature + 'Running QA on dataset {} of {} for feature "{}" ' + 'with source feature name "{}"'.format( + idf + 1, len(self.features), feature, source_feature, ) ) data_syn = self.get_dset_out(feature) data_syn = self.coarsen_data(idf, feature, data_syn) - data_true = self.input_handler[feature][...] + data_true = self.input_handler[source_feature][...] if data_syn.shape != data_true.shape: msg = ( diff --git a/sup3r/solar/solar.py b/sup3r/solar/solar.py index ed158bdbf..5f40381f3 100644 --- a/sup3r/solar/solar.py +++ b/sup3r/solar/solar.py @@ -138,11 +138,11 @@ def preflight(self): assert 'surface_pressure' in self.nsrdb.dsets assert isinstance(self.nsrdb_tslice, slice) - ti_gan = self.gan_data.time_index - delta = pd.Series(ti_gan[1:] - ti_gan[:-1]).mean().total_seconds() + delta = pd.Series(self.time_index[1:] - self.time_index[:-1]) + delta = delta.mean().total_seconds() msg = ( 'Its assumed that the sup3r GAN output solar data will be ' - 'hourly but received time index: {}'.format(ti_gan) + 'hourly but received time index: {}'.format(self.time_index) ) assert delta == 3600, msg @@ -305,9 +305,8 @@ def ghi(self): """ if self._ghi is None: logger.debug('Calculating GHI.') - self._ghi = ( - self.get_nsrdb_data('clearsky_ghi') * self.clearsky_ratio - ) + nsrdb_cs_ghi = self.get_nsrdb_data('clearsky_ghi') + self._ghi = nsrdb_cs_ghi * self.clearsky_ratio self._ghi[:, self.out_of_bounds] = 0 return self._ghi diff --git a/sup3r/utilities/cli.py b/sup3r/utilities/cli.py index 07eba4bf2..e704e8b43 100644 --- a/sup3r/utilities/cli.py +++ b/sup3r/utilities/cli.py @@ -73,10 +73,12 @@ def from_config(cls, module_name, module_class, ctx, config_file, verbose, cmd = module_class.get_node_cmd(config) if hardware_option.lower() in AVAILABLE_HARDWARE_OPTIONS: - cls.kickoff_slurm_job(module_name, ctx, pipeline_step, cmd, + cls.kickoff_slurm_job(module_name, ctx, cmd, + pipeline_step=pipeline_step, **exec_kwargs) else: - cls.kickoff_local_job(module_name, ctx, cmd, pipeline_step) + cls.kickoff_local_job(module_name, ctx, cmd, + pipeline_step=pipeline_step) @classmethod def from_config_preflight(cls, module_name, ctx, config_file, verbose):