Skip to content

Commit

Permalink
Merge pull request #203 from carbonplan/patch/qm-kwargs
Browse files Browse the repository at this point in the history 
pass bc_kwargs to QuantileMapper
  • Loading branch information
Joe Hamman authored Jun 2, 2022
2 parents 7ee609b + 33a796b commit 11949bf
Show file tree
Hide file tree
Showing 84 changed files with 167 additions and 146 deletions.
32 changes: 17 additions & 15 deletions cmip6_downscaling/methods/common/bias_correction.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,23 @@ def bias_correct_gcm_by_method(
if method == 'cunnane_transform':
return bc_kwargs['transformer_interp'].transform(gcm_pred)

elif method == 'z_score':
# transform gcm
sc = PointWiseDownscaler(model=StandardScaler(**bc_kwargs))
sc.fit(gcm_hist)
return sc.transform(gcm_pred)
elif method == 'quantile_mapper':
qm = PointWiseDownscaler(model=QuantileMapper(**bc_kwargs), dim='time')
qm.fit(obs)
return qm.transform(gcm_pred)

# TODO: test to see QuantileMappingReressor and TrendAwareQuantileMappingRegressor
# can handle multiple variables at once
elif method == 'quantile_map':
qm = PointWiseDownscaler(model=QuantileMappingReressor(**bc_kwargs), dim='time')
qm.fit(gcm_hist, obs)
return qm.predict(gcm_pred)

elif method == 'detrended_quantile_map':
qm = PointWiseDownscaler(
TrendAwareQuantileMappingRegressor(QuantileMappingReressor(**bc_kwargs))
Expand All @@ -67,16 +84,6 @@ def bias_correct_gcm_by_method(
elif method == 'none':
return gcm_pred

elif method == 'quantile_map':
qm = PointWiseDownscaler(model=QuantileMappingReressor(**bc_kwargs), dim='time')
qm.fit(gcm_hist, obs)
return qm.predict(gcm_pred)

elif method == 'quantile_mapper':
qm = PointWiseDownscaler(model=QuantileMapper(detrend=True), dim='time')
qm.fit(obs)
return qm.transform(gcm_pred)

elif method == 'quantile_transform':
# transform gcm
if 'n_quantiles' not in bc_kwargs:
Expand All @@ -85,11 +92,6 @@ def bias_correct_gcm_by_method(
qt = PointWiseDownscaler(model=QuantileTransformer(**bc_kwargs))
qt.fit(gcm_hist)
return qt.transform(gcm_pred)
elif method == 'z_score':
# transform gcm
sc = PointWiseDownscaler(model=StandardScaler(**bc_kwargs))
sc.fit(gcm_hist)
return sc.transform(gcm_pred)
else:
available_methods = [
'quantile_transform',
Expand Down
20 changes: 14 additions & 6 deletions cmip6_downscaling/methods/gard/tasks.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import zarr
from carbonplan_data.metadata import get_cf_global_attrs
from prefect import task
from scipy.special import cbrt
from skdownscale.pointwise_models import PointWiseDownscaler
from skdownscale.pointwise_models.utils import default_none_kwargs
from upath import UPath
Expand Down Expand Up @@ -85,19 +86,26 @@ def _fit_and_predict_wrapper(xtrain, ytrain, xpred, scrf, run_parameters, dim='t
.to_dataset(dim='variable')
.rename({'variable_0': run_parameters.variable})
)

# model definition
model = PointWiseDownscaler(
model=get_gard_model(run_parameters.model_type, run_parameters.model_params), dim=dim
)

# model fitting
model.fit(xtrain[run_parameters.variable], ytrain[run_parameters.variable])
if run_parameters.variable == 'pr':
model.fit(cbrt(xtrain[run_parameters.variable]), cbrt(ytrain[run_parameters.variable]))
out = model.predict(cbrt(bias_corrected_gcm_pred[run_parameters.variable])).to_dataset(
dim='variable'
)
out['pred'] = out['pred'] ** 3

# model prediction
out = model.predict(bias_corrected_gcm_pred[run_parameters.variable]).to_dataset(dim='variable')
else:
model.fit(xtrain[run_parameters.variable], ytrain[run_parameters.variable])
out = model.predict(bias_corrected_gcm_pred[run_parameters.variable]).to_dataset(
dim='variable'
)

downscaled = add_random_effects(out, scrf, run_parameters)
# model prediction
downscaled = add_random_effects(out, scrf.scrf, run_parameters)

return downscaled

Expand Down
13 changes: 10 additions & 3 deletions cmip6_downscaling/methods/gard/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
from typing import Any

import xarray as xr
from scipy.special import cbrt
from scipy.stats import norm as norm
from skdownscale.pointwise_models import AnalogRegression, PureAnalog, PureRegression

Expand Down Expand Up @@ -41,7 +42,7 @@ def get_gard_model(


def add_random_effects(
model_output: xr.Dataset, scrf: xr.Dataset, run_parameters: RunParameters
model_output: xr.Dataset, scrf: xr.DataArray, run_parameters: RunParameters
) -> xr.Dataset:
if run_parameters.model_params is not None:
thresh = run_parameters.model_params.get('thresh')
Expand All @@ -66,12 +67,18 @@ def add_random_effects(
r_normal = xr.apply_ufunc(
norm.ppf, new_uniform, dask='parallelized', output_dtypes=[new_uniform.dtype]
)

downscaled = model_output['pred'] + r_normal * model_output['prediction_error']
if run_parameters.variable == 'pr':
downscaled = (
cbrt(model_output['pred']) + (model_output['prediction_error'] * r_normal)
) ** 3
else:
downscaled = model_output['pred'] + r_normal * model_output['prediction_error']

# what do we do for thresholds like heat wave?
valids = xr.ufuncs.logical_or(mask, downscaled >= 0)
downscaled = downscaled.where(valids, 0)
downscaled = downscaled.where(downscaled >= 0, 0)

else:
downscaled = model_output['pred'] + scrf['scrf'] * model_output['prediction_error']

Expand Down
4 changes: 2 additions & 2 deletions ...csd/bcsd_ERA5_BCC-CSM2-MR_r1i1p1f1_historical_pr_-90_90_-180_180_1981_2010_1950_2014.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,9 @@
"lonmin": "-180",
"lonmax": "180",
"bias_correction_method": "",
"bias_correction_kwargs": {},
"bias_correction_kwargs": { "detrend": "False" },
"model_type": "",
"model_params": {},
"model_params": { "thresh": 0 },
"train_dates": ["1981", "2010"],
"predict_dates": ["1950", "2014"]
}
2 changes: 1 addition & 1 deletion ...bcsd_ERA5_BCC-CSM2-MR_r1i1p1f1_historical_tasmax_-90_90_-180_180_1981_2010_1950_2014.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
"lonmin": "-180",
"lonmax": "180",
"bias_correction_method": "",
"bias_correction_kwargs": {},
"bias_correction_kwargs": { "detrend": "True" },
"model_type": "",
"model_params": {},
"train_dates": ["1981", "2010"],
Expand Down
2 changes: 1 addition & 1 deletion ...bcsd_ERA5_BCC-CSM2-MR_r1i1p1f1_historical_tasmin_-90_90_-180_180_1981_2010_1950_2014.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
"lonmin": "-180",
"lonmax": "180",
"bias_correction_method": "",
"bias_correction_kwargs": {},
"bias_correction_kwargs": { "detrend": "True" },
"model_type": "",
"model_params": {},
"train_dates": ["1981", "2010"],
Expand Down
4 changes: 2 additions & 2 deletions ...gs/bcsd/bcsd_ERA5_BCC-CSM2-MR_r1i1p1f1_ssp245_pr_-90_90_-180_180_1981_2010_2015_2099.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,9 @@
"lonmin": "-180",
"lonmax": "180",
"bias_correction_method": "",
"bias_correction_kwargs": {},
"bias_correction_kwargs": { "detrend": "False" },
"model_type": "",
"model_params": {},
"model_params": { "thresh": 0 },
"train_dates": ["1981", "2010"],
"predict_dates": ["2015", "2099"]
}
2 changes: 1 addition & 1 deletion ...csd/bcsd_ERA5_BCC-CSM2-MR_r1i1p1f1_ssp245_tasmax_-90_90_-180_180_1981_2010_2015_2099.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
"lonmin": "-180",
"lonmax": "180",
"bias_correction_method": "",
"bias_correction_kwargs": {},
"bias_correction_kwargs": { "detrend": "True" },
"model_type": "",
"model_params": {},
"train_dates": ["1981", "2010"],
Expand Down
2 changes: 1 addition & 1 deletion ...csd/bcsd_ERA5_BCC-CSM2-MR_r1i1p1f1_ssp245_tasmin_-90_90_-180_180_1981_2010_2015_2099.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
"lonmin": "-180",
"lonmax": "180",
"bias_correction_method": "",
"bias_correction_kwargs": {},
"bias_correction_kwargs": { "detrend": "True" },
"model_type": "",
"model_params": {},
"train_dates": ["1981", "2010"],
Expand Down
4 changes: 2 additions & 2 deletions ...gs/bcsd/bcsd_ERA5_BCC-CSM2-MR_r1i1p1f1_ssp370_pr_-90_90_-180_180_1981_2010_2015_2099.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,9 @@
"lonmin": "-180",
"lonmax": "180",
"bias_correction_method": "",
"bias_correction_kwargs": {},
"bias_correction_kwargs": { "detrend": "False" },
"model_type": "",
"model_params": {},
"model_params": { "thresh": 0 },
"train_dates": ["1981", "2010"],
"predict_dates": ["2015", "2099"]
}
2 changes: 1 addition & 1 deletion ...csd/bcsd_ERA5_BCC-CSM2-MR_r1i1p1f1_ssp370_tasmax_-90_90_-180_180_1981_2010_2015_2099.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
"lonmin": "-180",
"lonmax": "180",
"bias_correction_method": "",
"bias_correction_kwargs": {},
"bias_correction_kwargs": { "detrend": "True" },
"model_type": "",
"model_params": {},
"train_dates": ["1981", "2010"],
Expand Down
2 changes: 1 addition & 1 deletion ...csd/bcsd_ERA5_BCC-CSM2-MR_r1i1p1f1_ssp370_tasmin_-90_90_-180_180_1981_2010_2015_2099.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
"lonmin": "-180",
"lonmax": "180",
"bias_correction_method": "",
"bias_correction_kwargs": {},
"bias_correction_kwargs": { "detrend": "True" },
"model_type": "",
"model_params": {},
"train_dates": ["1981", "2010"],
Expand Down
4 changes: 2 additions & 2 deletions ...gs/bcsd/bcsd_ERA5_BCC-CSM2-MR_r1i1p1f1_ssp585_pr_-90_90_-180_180_1981_2010_2015_2099.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,9 @@
"lonmin": "-180",
"lonmax": "180",
"bias_correction_method": "",
"bias_correction_kwargs": {},
"bias_correction_kwargs": { "detrend": "False" },
"model_type": "",
"model_params": {},
"model_params": { "thresh": 0 },
"train_dates": ["1981", "2010"],
"predict_dates": ["2015", "2099"]
}
2 changes: 1 addition & 1 deletion ...csd/bcsd_ERA5_BCC-CSM2-MR_r1i1p1f1_ssp585_tasmax_-90_90_-180_180_1981_2010_2015_2099.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
"lonmin": "-180",
"lonmax": "180",
"bias_correction_method": "",
"bias_correction_kwargs": {},
"bias_correction_kwargs": { "detrend": "True" },
"model_type": "",
"model_params": {},
"train_dates": ["1981", "2010"],
Expand Down
2 changes: 1 addition & 1 deletion ...csd/bcsd_ERA5_BCC-CSM2-MR_r1i1p1f1_ssp585_tasmin_-90_90_-180_180_1981_2010_2015_2099.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
"lonmin": "-180",
"lonmax": "180",
"bias_correction_method": "",
"bias_correction_kwargs": {},
"bias_correction_kwargs": { "detrend": "True" },
"model_type": "",
"model_params": {},
"train_dates": ["1981", "2010"],
Expand Down
4 changes: 2 additions & 2 deletions ...gs/bcsd/bcsd_ERA5_CanESM5_r1i1p1f1_historical_pr_-90_90_-180_180_1981_2010_1950_2014.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,9 @@
"lonmin": "-180",
"lonmax": "180",
"bias_correction_method": "",
"bias_correction_kwargs": {},
"bias_correction_kwargs": { "detrend": "False" },
"model_type": "",
"model_params": {},
"model_params": { "thresh": 0 },
"train_dates": ["1981", "2010"],
"predict_dates": ["1950", "2014"]
}
2 changes: 1 addition & 1 deletion ...csd/bcsd_ERA5_CanESM5_r1i1p1f1_historical_tasmax_-90_90_-180_180_1981_2010_1950_2014.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
"lonmin": "-180",
"lonmax": "180",
"bias_correction_method": "",
"bias_correction_kwargs": {},
"bias_correction_kwargs": { "detrend": "True" },
"model_type": "",
"model_params": {},
"train_dates": ["1981", "2010"],
Expand Down
2 changes: 1 addition & 1 deletion ...csd/bcsd_ERA5_CanESM5_r1i1p1f1_historical_tasmin_-90_90_-180_180_1981_2010_1950_2014.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
"lonmin": "-180",
"lonmax": "180",
"bias_correction_method": "",
"bias_correction_kwargs": {},
"bias_correction_kwargs": { "detrend": "True" },
"model_type": "",
"model_params": {},
"train_dates": ["1981", "2010"],
Expand Down
4 changes: 2 additions & 2 deletions ...onfigs/bcsd/bcsd_ERA5_CanESM5_r1i1p1f1_ssp245_pr_-90_90_-180_180_1981_2010_2015_2099.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,9 @@
"lonmin": "-180",
"lonmax": "180",
"bias_correction_method": "",
"bias_correction_kwargs": {},
"bias_correction_kwargs": { "detrend": "False" },
"model_type": "",
"model_params": {},
"model_params": { "thresh": 0 },
"train_dates": ["1981", "2010"],
"predict_dates": ["2015", "2099"]
}
2 changes: 1 addition & 1 deletion ...gs/bcsd/bcsd_ERA5_CanESM5_r1i1p1f1_ssp245_tasmax_-90_90_-180_180_1981_2010_2015_2099.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
"lonmin": "-180",
"lonmax": "180",
"bias_correction_method": "",
"bias_correction_kwargs": {},
"bias_correction_kwargs": { "detrend": "True" },
"model_type": "",
"model_params": {},
"train_dates": ["1981", "2010"],
Expand Down
2 changes: 1 addition & 1 deletion ...gs/bcsd/bcsd_ERA5_CanESM5_r1i1p1f1_ssp245_tasmin_-90_90_-180_180_1981_2010_2015_2099.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
"lonmin": "-180",
"lonmax": "180",
"bias_correction_method": "",
"bias_correction_kwargs": {},
"bias_correction_kwargs": { "detrend": "True" },
"model_type": "",
"model_params": {},
"train_dates": ["1981", "2010"],
Expand Down
4 changes: 2 additions & 2 deletions ...onfigs/bcsd/bcsd_ERA5_CanESM5_r1i1p1f1_ssp370_pr_-90_90_-180_180_1981_2010_2015_2099.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,9 @@
"lonmin": "-180",
"lonmax": "180",
"bias_correction_method": "",
"bias_correction_kwargs": {},
"bias_correction_kwargs": { "detrend": "False" },
"model_type": "",
"model_params": {},
"model_params": { "thresh": 0 },
"train_dates": ["1981", "2010"],
"predict_dates": ["2015", "2099"]
}
2 changes: 1 addition & 1 deletion ...gs/bcsd/bcsd_ERA5_CanESM5_r1i1p1f1_ssp370_tasmax_-90_90_-180_180_1981_2010_2015_2099.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
"lonmin": "-180",
"lonmax": "180",
"bias_correction_method": "",
"bias_correction_kwargs": {},
"bias_correction_kwargs": { "detrend": "True" },
"model_type": "",
"model_params": {},
"train_dates": ["1981", "2010"],
Expand Down
2 changes: 1 addition & 1 deletion ...gs/bcsd/bcsd_ERA5_CanESM5_r1i1p1f1_ssp370_tasmin_-90_90_-180_180_1981_2010_2015_2099.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
"lonmin": "-180",
"lonmax": "180",
"bias_correction_method": "",
"bias_correction_kwargs": {},
"bias_correction_kwargs": { "detrend": "True" },
"model_type": "",
"model_params": {},
"train_dates": ["1981", "2010"],
Expand Down
4 changes: 2 additions & 2 deletions ...onfigs/bcsd/bcsd_ERA5_CanESM5_r1i1p1f1_ssp585_pr_-90_90_-180_180_1981_2010_2015_2099.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,9 @@
"lonmin": "-180",
"lonmax": "180",
"bias_correction_method": "",
"bias_correction_kwargs": {},
"bias_correction_kwargs": { "detrend": "False" },
"model_type": "",
"model_params": {},
"model_params": { "thresh": 0 },
"train_dates": ["1981", "2010"],
"predict_dates": ["2015", "2099"]
}
2 changes: 1 addition & 1 deletion ...gs/bcsd/bcsd_ERA5_CanESM5_r1i1p1f1_ssp585_tasmax_-90_90_-180_180_1981_2010_2015_2099.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
"lonmin": "-180",
"lonmax": "180",
"bias_correction_method": "",
"bias_correction_kwargs": {},
"bias_correction_kwargs": { "detrend": "True" },
"model_type": "",
"model_params": {},
"train_dates": ["1981", "2010"],
Expand Down
2 changes: 1 addition & 1 deletion ...gs/bcsd/bcsd_ERA5_CanESM5_r1i1p1f1_ssp585_tasmin_-90_90_-180_180_1981_2010_2015_2099.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
"lonmin": "-180",
"lonmax": "180",
"bias_correction_method": "",
"bias_correction_kwargs": {},
"bias_correction_kwargs": { "detrend": "True" },
"model_type": "",
"model_params": {},
"train_dates": ["1981", "2010"],
Expand Down
4 changes: 2 additions & 2 deletions ...igs/bcsd/bcsd_ERA5_MIROC6_r1i1p1f1_historical_pr_-90_90_-180_180_1981_2010_1950_2014.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,9 @@
"lonmin": "-180",
"lonmax": "180",
"bias_correction_method": "",
"bias_correction_kwargs": {},
"bias_correction_kwargs": { "detrend": "False" },
"model_type": "",
"model_params": {},
"model_params": { "thresh": 0 },
"train_dates": ["1981", "2010"],
"predict_dates": ["1950", "2014"]
}
2 changes: 1 addition & 1 deletion ...bcsd/bcsd_ERA5_MIROC6_r1i1p1f1_historical_tasmax_-90_90_-180_180_1981_2010_1950_2014.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
"lonmin": "-180",
"lonmax": "180",
"bias_correction_method": "",
"bias_correction_kwargs": {},
"bias_correction_kwargs": { "detrend": "True" },
"model_type": "",
"model_params": {},
"train_dates": ["1981", "2010"],
Expand Down
Loading

1 comment on commit 11949bf

@vercel
Copy link

@vercel vercel bot commented on 11949bf Jun 2, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Successfully deployed to the following URLs:

cmip6-downscaling – ./

cmip6-downscaling.vercel.app
cmip6-downscaling-carbonplan.vercel.app
cmip6-downscaling.docs.carbonplan.org
cmip6-downscaling-git-main-carbonplan.vercel.app

Please sign in to comment.