diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml index 5944b0a0..02647a46 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml @@ -48,7 +48,7 @@ geovals_experiment: default_value: x0048-geovals geovals_provider: - default_value: ncdiag + default_value: x0048-geovals gradient_norm_reduction: default_value: 10e-5 @@ -85,7 +85,7 @@ obs_experiment: default_value: x0048 obs_provider: - default_value: ncdiag + default_value: x0048 observations: default_value: diff --git a/src/swell/configuration/jedi/interfaces/geos_ocean/model/r2d2.yaml b/src/swell/configuration/jedi/interfaces/geos_ocean/model/r2d2.yaml index 812dfdfc..8930eb56 100755 --- a/src/swell/configuration/jedi/interfaces/geos_ocean/model/r2d2.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_ocean/model/r2d2.yaml @@ -5,6 +5,7 @@ fetch: fc: - file_type: MOM.res filename: '{{cycle_dir}}/MOM6.res.{{local_background_time}}.nc' + domain: global store: fc: - file_type: MOM.res diff --git a/src/swell/deployment/bin/swell_create_experiment.py b/src/swell/deployment/bin/swell_create_experiment.py index 4a1ff6ba..01ec5c1a 100644 --- a/src/swell/deployment/bin/swell_create_experiment.py +++ b/src/swell/deployment/bin/swell_create_experiment.py @@ -20,7 +20,6 @@ from swell.deployment.prep_suite import prepare_cylc_suite_jinja2 from swell.swell_path import get_swell_path from swell.utilities.dictionary import dict_get -from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.logger import Logger from swell.utilities.welcome_message import write_welcome_message @@ -70,25 +69,6 @@ def main(config_file): if os.path.exists(os.path.join(swell_suite_path, 'eva')): copy_eva_files(logger, swell_suite_path, exp_suite_path) - # Create R2D2 database file - # ------------------------- - r2d2_local_path = dict_get(logger, experiment_dict, 'r2d2_local_path', None) - if r2d2_local_path is not None: - r2d2_conf_path = os.path.join(exp_suite_path, 'r2d2_config.yaml') - - # Write R2D2_CONFIG to modules - with open(os.path.join(exp_suite_path, 'modules'), 'a') as module_file: - module_file.write(f'export R2D2_CONFIG={r2d2_conf_path}') - - # Open the r2d2 file to dictionary - with open(r2d2_conf_path, 'r') as r2d2_file_open: - r2d2_file_str = r2d2_file_open.read() - r2d2_file_str = template_string_jinja2(logger, r2d2_file_str, experiment_dict) - r2d2_file_str = os.path.expandvars(r2d2_file_str) - - with open(r2d2_conf_path, 'w') as r2d2_file_open: - r2d2_file_open.write(r2d2_file_str) - # Set the swell paths in the modules file and create csh versions # --------------------------------------------------------------- template_modules_file(logger, experiment_dict, exp_suite_path) diff --git a/src/swell/deployment/platforms/generic/r2d2_config.yaml b/src/swell/deployment/platforms/generic/r2d2_config.yaml deleted file mode 100755 index fa3d4289..00000000 --- a/src/swell/deployment/platforms/generic/r2d2_config.yaml +++ /dev/null @@ -1,22 +0,0 @@ -databases: - - ${USER}: - class: LocalDB - root: {{r2d2_local_path}} - cache_fetch: false - - shared: - class: LocalDB - root: /home/${USER}/R2D2DataStore/Shared - cache_fetch: false - -# when fetching data, in which order should the databases accessed? -fetch_order: - - ${USER} - - shared - -# when storing data, in which order should the databases accessed? -store_order: - - ${USER} - -cache_name: ${USER} diff --git a/src/swell/deployment/platforms/generic/suite_questions.yaml b/src/swell/deployment/platforms/generic/suite_questions.yaml index 2793cbf0..79aab103 100644 --- a/src/swell/deployment/platforms/generic/suite_questions.yaml +++ b/src/swell/deployment/platforms/generic/suite_questions.yaml @@ -3,6 +3,3 @@ experiment_id: experiment_root: default_value: /home/${USER}/SwellExperiments - -r2d2_local_path: - default_value: /home/${USER}/R2D2DataStore/Local diff --git a/src/swell/deployment/platforms/generic/task_questions.yaml b/src/swell/deployment/platforms/generic/task_questions.yaml index 6bb85dac..1366a979 100644 --- a/src/swell/deployment/platforms/generic/task_questions.yaml +++ b/src/swell/deployment/platforms/generic/task_questions.yaml @@ -21,6 +21,24 @@ geos_restarts_directory: default_value: - /home/geos/restarts +r2d2_fetch_datastores: + default_value: + - swell-r2d2-${USER} + - swell-r2d2-archive + - swell-r2d2 + options: + - swell-r2d2-${USER} + - swell-r2d2-archive + - swell-r2d2 + +r2d2_store_datastores: + default_value: + - swell-r2d2-${USER} + - swell-r2d2 + options: + - swell-r2d2-${USER} + - swell-r2d2 + swell_static_files: default_value: This would need to be provided by user diff --git a/src/swell/deployment/platforms/nccs_discover/r2d2_config.yaml b/src/swell/deployment/platforms/nccs_discover/r2d2_config.yaml deleted file mode 100755 index 5c7c023a..00000000 --- a/src/swell/deployment/platforms/nccs_discover/r2d2_config.yaml +++ /dev/null @@ -1,22 +0,0 @@ -databases: - - ${USER}: - class: LocalDB - root: {{r2d2_local_path}} - cache_fetch: false - - gmao-shared: - class: LocalDB - root: /discover/nobackup/drholdaw/R2D2DataStore/Shared - cache_fetch: false - -# when fetching data, in which order should the databases accessed? -fetch_order: - - ${USER} - - gmao-shared - -# when storing data, in which order should the databases accessed? -store_order: - - ${USER} - -cache_name: ${USER} diff --git a/src/swell/deployment/platforms/nccs_discover/suite_questions.yaml b/src/swell/deployment/platforms/nccs_discover/suite_questions.yaml index 5344ca69..43a7c260 100644 --- a/src/swell/deployment/platforms/nccs_discover/suite_questions.yaml +++ b/src/swell/deployment/platforms/nccs_discover/suite_questions.yaml @@ -1,5 +1,2 @@ experiment_root: default_value: /discover/nobackup/${USER}/SwellExperiments - -r2d2_local_path: - default_value: /discover/nobackup/${USER}/R2D2DataStore/Local diff --git a/src/swell/deployment/platforms/nccs_discover/task_questions.yaml b/src/swell/deployment/platforms/nccs_discover/task_questions.yaml index 4dde31a2..194d99f5 100644 --- a/src/swell/deployment/platforms/nccs_discover/task_questions.yaml +++ b/src/swell/deployment/platforms/nccs_discover/task_questions.yaml @@ -19,6 +19,24 @@ geos_experiment_directory: geos_restarts_directory: default_value: restarts_20210620_000000 +r2d2_fetch_datastores: + default_value: + - swell-r2d2-discover-${USER} + - swell-r2d2-discover-archive + - swell-r2d2-discover + options: + - swell-r2d2-discover-${USER} + - swell-r2d2-discover-archive + - swell-r2d2-discover + +r2d2_store_datastores: + default_value: + - swell-r2d2-discover-${USER} + - swell-r2d2-discover + options: + - swell-r2d2-discover-${USER} + - swell-r2d2-discover + swell_static_files: default_value: /discover/nobackup/drholdaw/SwellStaticFiles diff --git a/src/swell/deployment/prep_exp_dirs.py b/src/swell/deployment/prep_exp_dirs.py index d45863c2..38860de7 100644 --- a/src/swell/deployment/prep_exp_dirs.py +++ b/src/swell/deployment/prep_exp_dirs.py @@ -46,13 +46,12 @@ def copy_platform_files(logger, exp_suite_path, platform=None): swell_lib_path = get_swell_path() platform_path = os.path.join(swell_lib_path, 'deployment', 'platforms', platform) - for s in ['modules', 'r2d2_config.yaml']: - src_file = os.path.split(s)[1] - src_path_file = os.path.join(platform_path, os.path.split(s)[0], src_file) - dst_path_file = os.path.join(exp_suite_path, '{}'.format(src_file)) - if os.path.exists(src_path_file): - logger.trace('Copying {} to {}'.format(src_path_file, dst_path_file)) - shutil.copy(src_path_file, dst_path_file) + src_path_file = os.path.join(platform_path, 'modules') + dst_path_file = os.path.join(exp_suite_path, 'modules') + + if os.path.exists(src_path_file): + logger.trace('Copying {} to {}'.format(src_path_file, dst_path_file)) + shutil.copy(src_path_file, dst_path_file) # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dvar/suite_questions.yaml b/src/swell/suites/3dvar/suite_questions.yaml index 71c889f6..f7598997 100644 --- a/src/swell/suites/3dvar/suite_questions.yaml +++ b/src/swell/suites/3dvar/suite_questions.yaml @@ -16,12 +16,6 @@ runahead_limit: prompt: Since this suite is non-cycling choose how many hours the workflow can run ahead? type: string -r2d2_local_path: - ask_question: False - default_value: defer_to_platform - prompt: Enter the path where R2D2 will store experiment output - type: string - cycle_times: ask_question: True default_value: defer_to_model diff --git a/src/swell/suites/3dvar_cycle/suite_questions.yaml b/src/swell/suites/3dvar_cycle/suite_questions.yaml index 3e3090ee..487ccda7 100644 --- a/src/swell/suites/3dvar_cycle/suite_questions.yaml +++ b/src/swell/suites/3dvar_cycle/suite_questions.yaml @@ -10,12 +10,6 @@ final_cycle_point: prompt: What is the time of the final cycle (middle of the window)? type: iso-datetime -r2d2_local_path: - ask_question: False - default_value: defer_to_platform - prompt: Enter the path where R2D2 will store experiment output - type: string - cycle_times: ask_question: True default_value: defer_to_model diff --git a/src/swell/suites/convert_ncdiags/suite_questions.yaml b/src/swell/suites/convert_ncdiags/suite_questions.yaml index a5c1d9e8..d0e6f305 100644 --- a/src/swell/suites/convert_ncdiags/suite_questions.yaml +++ b/src/swell/suites/convert_ncdiags/suite_questions.yaml @@ -16,12 +16,6 @@ runahead_limit: prompt: Since this suite is non-cycling choose how many hours the workflow can run ahead? type: string -r2d2_local_path: - ask_question: False - default_value: defer_to_platform - prompt: Enter the path where R2D2 will store experiment output - type: string - cycle_times: ask_question: True default_value: defer_to_model diff --git a/src/swell/suites/forecast_geos/suite_questions.yaml b/src/swell/suites/forecast_geos/suite_questions.yaml index af75f762..f41dffb7 100644 --- a/src/swell/suites/forecast_geos/suite_questions.yaml +++ b/src/swell/suites/forecast_geos/suite_questions.yaml @@ -10,12 +10,6 @@ final_cycle_point: prompt: What is the time of the final cycle (middle of the window)? type: iso-datetime -r2d2_local_path: - ask_question: False - default_value: defer_to_platform - prompt: Enter the path where R2D2 will store experiment output - type: string - cycle_times: ask_question: True default_value: diff --git a/src/swell/suites/geosadas/suite_questions.yaml b/src/swell/suites/geosadas/suite_questions.yaml index 3e3090ee..487ccda7 100644 --- a/src/swell/suites/geosadas/suite_questions.yaml +++ b/src/swell/suites/geosadas/suite_questions.yaml @@ -10,12 +10,6 @@ final_cycle_point: prompt: What is the time of the final cycle (middle of the window)? type: iso-datetime -r2d2_local_path: - ask_question: False - default_value: defer_to_platform - prompt: Enter the path where R2D2 will store experiment output - type: string - cycle_times: ask_question: True default_value: defer_to_model diff --git a/src/swell/suites/hofx/suite_questions.yaml b/src/swell/suites/hofx/suite_questions.yaml index 71c889f6..f7598997 100644 --- a/src/swell/suites/hofx/suite_questions.yaml +++ b/src/swell/suites/hofx/suite_questions.yaml @@ -16,12 +16,6 @@ runahead_limit: prompt: Since this suite is non-cycling choose how many hours the workflow can run ahead? type: string -r2d2_local_path: - ask_question: False - default_value: defer_to_platform - prompt: Enter the path where R2D2 will store experiment output - type: string - cycle_times: ask_question: True default_value: defer_to_model diff --git a/src/swell/suites/ufo_testing/suite_questions.yaml b/src/swell/suites/ufo_testing/suite_questions.yaml index 71c889f6..f7598997 100644 --- a/src/swell/suites/ufo_testing/suite_questions.yaml +++ b/src/swell/suites/ufo_testing/suite_questions.yaml @@ -16,12 +16,6 @@ runahead_limit: prompt: Since this suite is non-cycling choose how many hours the workflow can run ahead? type: string -r2d2_local_path: - ask_question: False - default_value: defer_to_platform - prompt: Enter the path where R2D2 will store experiment output - type: string - cycle_times: ask_question: True default_value: defer_to_model diff --git a/src/swell/tasks/get_background.py b/src/swell/tasks/get_background.py index a7196ee7..b10deea3 100644 --- a/src/swell/tasks/get_background.py +++ b/src/swell/tasks/get_background.py @@ -7,13 +7,11 @@ # -------------------------------------------------------------------------------------------------- - from swell.tasks.base.task_base import taskBase +from swell.utilities.store_fetch import fetch import isodate import os -from r2d2 import fetch - # -------------------------------------------------------------------------------------------------- @@ -50,6 +48,12 @@ def execute(self): window_offset = self.config.window_offset() window_type = self.config.window_type() + r2d2_fetch_datastores = self.config.r2d2_fetch_datastores(['swell-r2d2-archive']) + r2d2_fetch_datastores = [r.replace("${USER}", os.getenv('USER')) + for r in r2d2_fetch_datastores] + + self.logger.info("Fetching from R2D2 data_stores: " + ', '.join(r2d2_fetch_datastores)) + # Get window parameters local_background_time = self.da_window_params.local_background_time(window_offset, window_type) @@ -118,6 +122,9 @@ def execute(self): # Get r2d2 dictionary r2d2_dict = self.jedi_rendering.render_interface_model('r2d2') + # To force localhost r2d2 + # os.environ['R2D2_HOST'] = 'localhost' + # Loop over fc # ------------ for fc in r2d2_dict['fetch']['fc']: @@ -126,6 +133,7 @@ def execute(self): # -------------------- file_type = fc['file_type'] target_file_template = fc['filename'] + domain = fc.get('domain', '') # Loop over background steps # -------------------- @@ -139,18 +147,22 @@ def execute(self): # --------------------------------------------------- target_file = background_time.strftime(target_file_template) - fetch( - date=forecast_start_time, - target_file=target_file, - model=r2d2_model_dict[model_component], - file_type=file_type, - fc_date_rendering='analysis', - step=bkg_step, - resolution=horizontal_resolution, - type='fc', - experiment=background_experiment) + target_dir = os.path.dirname(target_file) + os.makedirs(target_dir, exist_ok=True) + + fetched_from = fetch(r2d2_fetch_datastores, + item='forecast', + target_file=target_file, + model=r2d2_model_dict[model_component], + experiment=background_experiment, + file_extension="nc", + resolution=horizontal_resolution, + domain=domain, + file_type=file_type, + step=bkg_step, + date=forecast_start_time) + + self.logger.info("Fetched R2D2 data from " + fetched_from + " to " + target_file) # Change permission os.chmod(target_file, 0o644) - -# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/get_observations.py b/src/swell/tasks/get_observations.py index e382cd7f..ad9a0dff 100644 --- a/src/swell/tasks/get_observations.py +++ b/src/swell/tasks/get_observations.py @@ -7,12 +7,10 @@ # -------------------------------------------------------------------------------------------------- - -import os - from swell.tasks.base.task_base import taskBase -from r2d2 import fetch +from swell.utilities.store_fetch import fetch +import os # -------------------------------------------------------------------------------------------------- @@ -43,6 +41,12 @@ def execute(self): crtm_coeff_dir = self.config.crtm_coeff_dir(None) window_offset = self.config.window_offset() + r2d2_fetch_datastores = self.config.r2d2_fetch_datastores(['swell-r2d2-archive']) + r2d2_fetch_datastores = [r.replace("${USER}", os.getenv('USER')) + for r in r2d2_fetch_datastores] + + self.logger.info("Fetching from R2D2 data_stores: " + ', '.join(r2d2_fetch_datastores)) + # Get window begin time window_begin = self.da_window_params.window_begin(window_offset) background_time = self.da_window_params.background_time(window_offset, @@ -53,6 +57,9 @@ def execute(self): self.jedi_rendering.add_key('crtm_coeff_dir', crtm_coeff_dir) self.jedi_rendering.add_key('window_begin', window_begin) + # To force localhost r2d2 + # os.environ['R2D2_HOST'] = 'localhost' + # Loop over observation operators # ------------------------------- for observation in observations: @@ -64,15 +71,23 @@ def execute(self): # Fetch observation files # ----------------------- target_file = observation_dict['obs space']['obsdatain']['engine']['obsfile'] + self.logger.info(f'Processing observation file {target_file}') - fetch(date=window_begin, - target_file=target_file, - provider=obs_provider, - obs_type=observation, - time_window=window_length, - type='ob', - experiment=obs_experiment) + target_dir = os.path.dirname(target_file) + os.makedirs(target_dir, exist_ok=True) + file_extension = os.path.splitext(target_file)[1].replace(".", "") + + fetched_from = fetch(r2d2_fetch_datastores, + item='observation', + target_file=target_file, + provider=obs_provider, + observation_type=observation, + file_extension=file_extension, + window_start=window_begin, + window_length=window_length) + + self.logger.info("Fetched R2D2 data from " + fetched_from + " to " + target_file) # Change permission os.chmod(target_file, 0o644) @@ -116,30 +131,48 @@ def execute(self): target_file = observation_dict['obs bias']['input file'] self.logger.info(f'Processing satellite bias file {target_file}') - fetch(date=background_time, - target_file=target_file, - provider='gsi', - obs_type=observation, - type='bc', - experiment=obs_experiment, - file_type='satbias') + target_dir = os.path.dirname(target_file) + os.makedirs(target_dir, exist_ok=True) + file_extension = os.path.splitext(target_file)[1].replace(".", "") + + fetched_from = fetch(r2d2_fetch_datastores, + item='bias_correction', + target_file=target_file, + model='geos_atmosphere', + experiment=obs_experiment, + provider='gsi', + observation_type=observation, + file_extension=file_extension, + file_type='satbias', + date=background_time) + + self.logger.info("Fetched R2D2 data from " + fetched_from + " to " + target_file) # Change permission os.chmod(target_file, 0o644) # Satellite time lapse # -------------------- - for target_file in self.get_tlapse_files(observation_dict): + for target_file in list(set(list(self.get_tlapse_files(observation_dict)))): self.logger.info(f'Processing time lapse file {target_file}') - fetch(date=background_time, - target_file=target_file, - provider='gsi', - obs_type=observation, - type='bc', - experiment=obs_experiment, - file_type='tlapse') + target_dir = os.path.dirname(target_file) + os.makedirs(target_dir, exist_ok=True) + file_extension = os.path.splitext(target_file)[1].replace(".", "") + + fetched_from = fetch(r2d2_fetch_datastores, + item='bias_correction', + target_file=target_file, + model='geos_atmosphere', + experiment=obs_experiment, + provider='gsi', + observation_type=observation, + file_extension=file_extension, + file_type='tlapse', + date=background_time) + + self.logger.info("Fetched R2D2 data from " + fetched_from + " to " + target_file) # Change permission os.chmod(target_file, 0o644) diff --git a/src/swell/tasks/save_obs_diags.py b/src/swell/tasks/save_obs_diags.py index b3b15b90..056d315c 100644 --- a/src/swell/tasks/save_obs_diags.py +++ b/src/swell/tasks/save_obs_diags.py @@ -7,9 +7,10 @@ # -------------------------------------------------------------------------------------------------- -import os from swell.tasks.base.task_base import taskBase -from r2d2 import store +from swell.utilities.store_fetch import store + +import os # -------------------------------------------------------------------------------------------------- @@ -29,11 +30,21 @@ def execute(self): observations = self.config.observations() window_offset = self.config.window_offset() + r2d2_store_datastores = self.config.r2d2_store_datastores(['swell-r2d2']) + r2d2_store_datastores = [r.replace("${USER}", os.getenv('USER')) + for r in r2d2_store_datastores] + limit_store = self.config.limit_r2d2_storing(True) + + self.logger.info("Storing to R2D2 data_stores: " + ', '.join(r2d2_store_datastores)) + # Get window beginning window_begin = self.da_window_params.window_begin(window_offset) background_time = self.da_window_params.background_time(window_offset, background_time_offset) + # To force localhost r2d2 + # os.environ['R2D2_HOST'] = 'localhost' + # Create templates dictionary self.jedi_rendering.add_key('background_time', background_time) self.jedi_rendering.add_key('crtm_coeff_dir', crtm_coeff_dir) @@ -53,16 +64,26 @@ def execute(self): # Check for need to add 0000 to the file if not os.path.exists(obs_path_file): + obs_path_file_name, obs_path_file_ext = os.path.splitext(obs_path_file) obs_path_file_0000 = obs_path_file_name + '_0000' + obs_path_file_ext + if not os.path.exists(obs_path_file_0000): self.logger.abort(f'No observation file found for {obs_path_file} or ' + f'{obs_path_file_0000}') + obs_path_file = obs_path_file_0000 - store(date=window_begin, - provider='ncdiag', - source_file=obs_path_file, - obs_type=name, - type='ob', - experiment=self.experiment_id()) + file_extension = os.path.splitext(obs_path_file)[1].replace(".", "") + + stored_to = store(r2d2_store_datastores, + limit_one=limit_store, + item='observation', + source_file=obs_path_file, + provider='x0044', + observation_type=name, + file_extension=file_extension, + window_start=window_begin, + window_length=window_offset) + + self.logger.info("Stored R2D2 data to " + str(stored_to)) diff --git a/src/swell/tasks/store_background.py b/src/swell/tasks/store_background.py index f6e76f8f..5393f9ae 100644 --- a/src/swell/tasks/store_background.py +++ b/src/swell/tasks/store_background.py @@ -11,8 +11,7 @@ from datetime import datetime as dt import isodate import os -from r2d2 import store - +from r2d2 import R2D2Data from swell.tasks.base.task_base import taskBase from swell.utilities.datetime import datetime_formats @@ -101,11 +100,14 @@ def execute(self): # Get r2d2 dictionary r2d2_dict = self.jedi_rendering.render_interface_model('r2d2') + # To force localhost r2d2 + # os.environ['R2D2_HOST'] = 'localhost' + # Loop over fc for fc in r2d2_dict['store']['fc']: - # Reset target file - target_file_template = os.path.split(background_dict['filename'])[1] + # Reset source file + source_file_template = os.path.split(background_dict['filename'])[1] # Datetime format to use user_date_format = fc['user_date_format'] @@ -113,8 +115,8 @@ def execute(self): # Loop over file types for file_type in fc['file_type']: - # Replace filetype in target_file_template - target_file_type_template = target_file_template.replace("$(file_type)", file_type) + # Replace filetype in source_file_template + source_file_type_template = source_file_template.replace("$(file_type)", file_type) # Looop over background steps for bkg_step in bkg_steps: @@ -123,17 +125,20 @@ def execute(self): background_time = forecast_start_time + isodate.parse_duration(bkg_step) valid_time_str = background_time.strftime(user_date_format) - # Set the target file name - target_file = target_file_type_template.replace("$(valid_date)", valid_time_str) - target_file = os.path.join(self.cycle_dir(), target_file) + # Set the source file name + source_file = source_file_type_template.replace("$(valid_date)", valid_time_str) + source_file = os.path.join(cfg.get('cycle_dir'), source_file) + + file_extension = os.path.splitext(source_file)[1].replace(".", "") # Perform the store - store(date=forecast_start_time, - source_file=target_file, - model='geos', - file_type='bkg', - fc_date_rendering='analysis', - step=bkg_step, - resolution=self.config.horizontal_resolution(), - type='fc', - experiment=background_experiment) + R2D2Data.store(item='forecast', + source_file=source_file, + data_store='swell-r2d2', + model='geos', + experiment=background_experiment, + file_extension=file_extension, + resolution=cfg.get('horizontal_resolution'), + file_type='bkg', + step=bkg_step, + date=forecast_start_time) diff --git a/src/swell/tasks/task_questions.yaml b/src/swell/tasks/task_questions.yaml index d7bf42e6..538c7a02 100644 --- a/src/swell/tasks/task_questions.yaml +++ b/src/swell/tasks/task_questions.yaml @@ -328,7 +328,6 @@ horizontal_resolution: - RunJediLetkfExecutable - RunJediVariationalExecutable - StageJedi - - StoreBackground type: string-drop-list jedi_bkg_filename_template: @@ -370,6 +369,14 @@ jedi_forecast_model: - RunJediVariationalExecutable type: string-drop-list +limit_r2d2_storing: + ask_question: true + default_value: true + prompt: Question + tasks: + - SaveObsDiags + type: boolean + minimizer: ask_question: false default_value: defer_to_model @@ -515,6 +522,25 @@ produce_geovals: - GsiNcdiagToIoda type: boolean +r2d2_fetch_datastores: + ask_question: false + default_value: defer_to_platform + options: defer_to_platform + prompt: In what order do you want to search R2D2 data_stores for fetching? + tasks: + - GetBackground + - GetObservations + type: string-check-list + +r2d2_store_datastores: + ask_question: false + default_value: defer_to_platform + options: defer_to_platform + prompt: In which R2D2 data_stores do you want to store new data? + tasks: + - SaveObsDiags + type: string-check-list + swell_static_files: ask_question: false default_value: defer_to_platform diff --git a/src/swell/utilities/store_fetch.py b/src/swell/utilities/store_fetch.py new file mode 100644 index 00000000..906f4838 --- /dev/null +++ b/src/swell/utilities/store_fetch.py @@ -0,0 +1,95 @@ +# (C) Copyright 2023 United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + +from r2d2 import R2D2Data +import r2d2.error as err + + +# -------------------------------------------------------------------------------------------------- +def fetch(data_stores, **fetch_kwargs): + + # Track whether a fetch was successful + fetch_success = False + + # Gather error messages for each fetch attempt + failure_messages = dict.fromkeys(data_stores) + + # Loop over each data store + for data_store in data_stores: + + print("******", data_store, "*******") + + # Try to Fetch + try: + R2D2Data.fetch(data_store=data_store, **fetch_kwargs) + + except (err.RegistrationNotFound, err.RecordNotFound) as r2d2error: + failure_messages[data_store] = str(r2d2error) + pass + + else: + fetch_success = True + fetched_from = data_store + break + + if fetch_success is False: + + error_message = "Failed to fetch R2D2 data.\n" + for k, v in failure_messages.items(): + error_message += f'{k}: {v}\n' + + # Is there a more appropriate error than ValueError? + raise ValueError(error_message) + + return fetched_from + + +# -------------------------------------------------------------------------------------------------- +def store(data_stores, limit_one=True, **store_kwargs): + + """Very similar to Fetch, but + 1. Has option to store into every data_store or limit to first found. + 2. Ignores r2d2.error.RecordNotFound as it is irrelevant to storing. + 3. Returns indices of stored data. + """ + + # Track whether at least 1 store is successful + store_success = False + + # Gather error messages for each store attempt + failure_messages = dict.fromkeys(data_stores) + + # Return index (int) of stored item + indices = dict.fromkeys(data_stores) + + for data_store in data_stores: + + print("******", data_store, "*******") + + try: + indices[data_store] = R2D2Data.store(data_store=data_store, **store_kwargs) + + except err.RegistrationNotFound as r2d2error: + failure_messages[data_store] = str(r2d2error) + pass + + else: + store_success = True + if limit_one is True: + break + + if store_success is False: + + error_message = "Could not store data into any R2D2 data_store.\n" + for k, v in failure_messages.items(): + error_message += f'{k}: {v}\n' + + raise ValueError(error_message) + + return indices