diff --git a/config_src/drivers/nuopc_cap/mom_cap.F90 b/config_src/drivers/nuopc_cap/mom_cap.F90 index 76d541813e..f74aa45c77 100644 --- a/config_src/drivers/nuopc_cap/mom_cap.F90 +++ b/config_src/drivers/nuopc_cap/mom_cap.F90 @@ -28,9 +28,11 @@ module MOM_cap_mod use MOM_cap_methods, only: mom_import, mom_export, mom_set_geomtype, mod2med_areacor use MOM_cap_methods, only: med2mod_areacor, state_diagnose use MOM_cap_methods, only: ChkErr +use MOM_ensemble_manager, only: ensemble_manager_init #ifdef CESMCOUPLED use shr_log_mod, only: shr_log_setLogUnit +use nuopc_shr_methods, only: get_component_instance #endif use time_utils_mod, only: esmf2fms_time @@ -146,7 +148,8 @@ module MOM_cap_mod logical :: cesm_coupled = .false. type(ESMF_GeomType_Flag) :: geomtype #endif -character(len=8) :: restart_mode = 'alarms' +character(len=8) :: restart_mode = 'alarms' +character(len=16) :: inst_suffix = '' contains @@ -422,6 +425,8 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc) ! (same as restartfile if single restart file) character(len=*), parameter :: subname='(MOM_cap:InitializeAdvertise)' character(len=32) :: calendar + character(len=:), allocatable :: rpointer_filename + integer :: inst_index !-------------------------------- rc = ESMF_SUCCESS @@ -451,6 +456,13 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc) CALL ESMF_TimeIntervalGet(TINT, S=DT_OCEAN, RC=rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return +#ifdef CESMCOUPLED + call get_component_instance(gcomp, inst_suffix, inst_index, rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + call ensemble_manager_init(inst_suffix) + rpointer_filename = 'rpointer.ocn'//trim(inst_suffix) +#endif + ! reset shr logging to my log file if (localPet==0) then call NUOPC_CompAttributeGet(gcomp, name="diro", & @@ -460,11 +472,19 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc) isPresent=isPresentLogfile, rc=rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return if (isPresentDiro .and. isPresentLogfile) then - call NUOPC_CompAttributeGet(gcomp, name="diro", value=diro, rc=rc) - if (ChkErr(rc,__LINE__,u_FILE_u)) return - call NUOPC_CompAttributeGet(gcomp, name="logfile", value=logfile, rc=rc) - if (ChkErr(rc,__LINE__,u_FILE_u)) return - open(newunit=stdout,file=trim(diro)//"/"//trim(logfile)) + call NUOPC_CompAttributeGet(gcomp, name="diro", value=diro, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + call NUOPC_CompAttributeGet(gcomp, name="logfile", value=logfile, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + + if (cesm_coupled) then + ! Multiinstance logfile name needs a correction + if(logfile(4:4) == '_') then + logfile = logfile(1:3)//trim(inst_suffix)//logfile(9:) + endif + endif + + open(newunit=stdout,file=trim(diro)//"/"//trim(logfile)) else stdout = output_unit endif @@ -521,12 +541,6 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc) time0 = set_date (YEAR,MONTH,DAY,HOUR,MINUTE,SECOND) - - ! rsd need to figure out how to get this without share code - !call shr_nuopc_get_component_instance(gcomp, inst_suffix, inst_index) - !inst_name = "OCN"//trim(inst_suffix) - - if (is_root_pe()) then write(stdout,*) subname//'start time: y,m,d-',year,month,day,'h,m,s=',hour,minute,second endif @@ -581,9 +595,9 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc) if (localPet == 0) then ! this hard coded for rpointer.ocn right now - open(newunit=readunit, file='rpointer.ocn', form='formatted', status='old', iostat=iostat) + open(newunit=readunit, file=rpointer_filename, form='formatted', status='old', iostat=iostat) if (iostat /= 0) then - call ESMF_LogSetError(ESMF_RC_FILE_OPEN, msg=subname//' ERROR opening rpointer.ocn', & + call ESMF_LogSetError(ESMF_RC_FILE_OPEN, msg=subname//' ERROR opening '//rpointer_filename, & line=__LINE__, file=u_FILE_u, rcToReturn=rc) return endif @@ -593,7 +607,7 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc) if (len(trim(restartfiles))>1 .and. iostat<0) then exit ! done reading restart files list. else - call ESMF_LogSetError(ESMF_RC_FILE_READ, msg=subname//' ERROR reading rpointer.ocn', & + call ESMF_LogSetError(ESMF_RC_FILE_READ, msg=subname//' ERROR reading '//rpointer_filename, & line=__LINE__, file=u_FILE_u, rcToReturn=rc) return endif @@ -616,7 +630,12 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc) endif ocean_public%is_ocean_pe = .true. - call ocean_model_init(ocean_public, ocean_state, time0, time_start, input_restart_file=trim(adjustl(restartfiles))) + if (cesm_coupled .and. len_trim(inst_suffix)>0) then + call ocean_model_init(ocean_public, ocean_state, time0, time_start, & + input_restart_file=trim(adjustl(restartfiles)), inst_index=inst_index) + else + call ocean_model_init(ocean_public, ocean_state, time0, time_start, input_restart_file=trim(adjustl(restartfiles))) + endif ! GMM, this call is not needed in CESM. Check with EMC if it can be deleted. call ocean_model_flux_init(ocean_state) @@ -1489,6 +1508,7 @@ subroutine ModelAdvance(gcomp, rc) character(len=128) :: fldname character(len=*),parameter :: subname='(MOM_cap:ModelAdvance)' character(len=8) :: suffix + character(len=:), allocatable :: rpointer_filename integer :: num_rest_files rc = ESMF_SUCCESS @@ -1658,6 +1678,8 @@ subroutine ModelAdvance(gcomp, rc) call ESMF_VMGet(vm, localPet=localPet, rc=rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return + rpointer_filename = 'rpointer.ocn'//trim(inst_suffix) + write(restartname,'(A,".mom6.r.",I4.4,"-",I2.2,"-",I2.2,"-",I5.5)') & trim(casename), year, month, day, seconds call ESMF_LogWrite("MOM_cap: Writing restart : "//trim(restartname), ESMF_LOGMSG_INFO) @@ -1665,13 +1687,17 @@ subroutine ModelAdvance(gcomp, rc) call ocean_model_restart(ocean_state, restartname=restartname, num_rest_files=num_rest_files) if (localPet == 0) then ! Write name of restart file in the rpointer file - this is currently hard-coded for the ocean - open(newunit=writeunit, file='rpointer.ocn', form='formatted', status='unknown', iostat=iostat) + open(newunit=writeunit, file=rpointer_filename, form='formatted', status='unknown', iostat=iostat) if (iostat /= 0) then call ESMF_LogSetError(ESMF_RC_FILE_OPEN, & - msg=subname//' ERROR opening rpointer.ocn', line=__LINE__, file=u_FILE_u, rcToReturn=rc) + msg=subname//' ERROR opening '//rpointer_filename, line=__LINE__, file=u_FILE_u, rcToReturn=rc) return endif - write(writeunit,'(a)') trim(restartname)//'.nc' + if (len_trim(inst_suffix) == 0) then + write(writeunit,'(a)') trim(restartname)//'.nc' + else + write(writeunit,'(a)') trim(restartname)//'.'//trim(inst_suffix)//'.nc' + endif if (num_rest_files > 1) then ! append i.th restart file name to rpointer diff --git a/config_src/drivers/nuopc_cap/mom_ocean_model_nuopc.F90 b/config_src/drivers/nuopc_cap/mom_ocean_model_nuopc.F90 index e58c2796c8..9c81a67202 100644 --- a/config_src/drivers/nuopc_cap/mom_ocean_model_nuopc.F90 +++ b/config_src/drivers/nuopc_cap/mom_ocean_model_nuopc.F90 @@ -231,7 +231,7 @@ module MOM_ocean_model_nuopc !! This subroutine initializes both the ocean state and the ocean surface type. !! Because of the way that indicies and domains are handled, Ocean_sfc must have !! been used in a previous call to initialize_ocean_type. -subroutine ocean_model_init(Ocean_sfc, OS, Time_init, Time_in, gas_fields_ocn, input_restart_file) +subroutine ocean_model_init(Ocean_sfc, OS, Time_init, Time_in, gas_fields_ocn, input_restart_file, inst_index) type(ocean_public_type), target, & intent(inout) :: Ocean_sfc !< A structure containing various publicly !! visible ocean surface properties after initialization, @@ -248,6 +248,7 @@ subroutine ocean_model_init(Ocean_sfc, OS, Time_init, Time_in, gas_fields_ocn, i !! tracer fluxes, and can be used to spawn related !! internal variables in the ice model. character(len=*), optional, intent(in) :: input_restart_file !< If present, name of restart file to read + integer, optional :: inst_index !< Ensemble index provided by the cap (instead of FMS ensemble manager) ! Local variables real :: Rho0 ! The Boussinesq ocean density, in kg m-3. @@ -285,7 +286,7 @@ subroutine ocean_model_init(Ocean_sfc, OS, Time_init, Time_in, gas_fields_ocn, i OS%restart_CSp, Time_in, offline_tracer_mode=OS%offline_tracer_mode, & input_restart_file=input_restart_file, & diag_ptr=OS%diag, count_calls=.true., tracer_flow_CSp=OS%tracer_flow_CSp, & - waves_CSp=OS%Waves) + waves_CSp=OS%Waves, ensemble_num=inst_index) call get_MOM_state_elements(OS%MOM_CSp, G=OS%grid, GV=OS%GV, US=OS%US, C_p=OS%C_p, & C_p_scaled=OS%fluxes%C_p, use_temp=use_temperature) diff --git a/config_src/infra/FMS1/MOM_ensemble_manager_infra.F90 b/config_src/infra/FMS1/MOM_ensemble_manager_infra.F90 index 66bbb86e2f..3ab9d591da 100644 --- a/config_src/infra/FMS1/MOM_ensemble_manager_infra.F90 +++ b/config_src/infra/FMS1/MOM_ensemble_manager_infra.F90 @@ -9,6 +9,7 @@ module MOM_ensemble_manager_infra use ensemble_manager_mod, only : FMS_get_ensemble_size => get_ensemble_size use ensemble_manager_mod, only : FMS_get_ensemble_pelist => get_ensemble_pelist use ensemble_manager_mod, only : FMS_get_ensemble_filter_pelist => get_ensemble_filter_pelist +use fms_io_mod, only : fms_io_set_filename_appendix=>set_filename_appendix implicit none ; private @@ -20,9 +21,15 @@ module MOM_ensemble_manager_infra !> Initializes the ensemble manager which divides available resources !! in order to concurrently execute an ensemble of model realizations. -subroutine ensemble_manager_init() - - call FMS_ensemble_manager_init() +subroutine ensemble_manager_init(ensemble_suffix) + character(len=*), optional, intent(in) :: ensemble_suffix !> Ensemble suffix provided by the cap. This may be + !! provided to bypass FMS ensemble manager. + + if (present(ensemble_suffix)) then + call fms_io_set_filename_appendix(trim(ensemble_suffix)) + else + call FMS_ensemble_manager_init() + endif end subroutine ensemble_manager_init diff --git a/config_src/infra/FMS2/MOM_ensemble_manager_infra.F90 b/config_src/infra/FMS2/MOM_ensemble_manager_infra.F90 index 66bbb86e2f..c9eb067e54 100644 --- a/config_src/infra/FMS2/MOM_ensemble_manager_infra.F90 +++ b/config_src/infra/FMS2/MOM_ensemble_manager_infra.F90 @@ -9,6 +9,8 @@ module MOM_ensemble_manager_infra use ensemble_manager_mod, only : FMS_get_ensemble_size => get_ensemble_size use ensemble_manager_mod, only : FMS_get_ensemble_pelist => get_ensemble_pelist use ensemble_manager_mod, only : FMS_get_ensemble_filter_pelist => get_ensemble_filter_pelist +use fms2_io_mod, only : fms2_io_set_filename_appendix=>set_filename_appendix +use fms_io_mod, only : fms_io_set_filename_appendix=>set_filename_appendix implicit none ; private @@ -20,9 +22,16 @@ module MOM_ensemble_manager_infra !> Initializes the ensemble manager which divides available resources !! in order to concurrently execute an ensemble of model realizations. -subroutine ensemble_manager_init() - - call FMS_ensemble_manager_init() +subroutine ensemble_manager_init(ensemble_suffix) + character(len=*), optional, intent(in) :: ensemble_suffix !> Ensemble suffix provided by the cap. This may be + !! provided to bypass FMS ensemble manager. + + if (present(ensemble_suffix)) then + call fms2_io_set_filename_appendix(trim(ensemble_suffix)) + call fms_io_set_filename_appendix(trim(ensemble_suffix)) + else + call FMS_ensemble_manager_init() + endif end subroutine ensemble_manager_init diff --git a/src/core/MOM.F90 b/src/core/MOM.F90 index c61f130ef7..54d2310cfe 100644 --- a/src/core/MOM.F90 +++ b/src/core/MOM.F90 @@ -1830,7 +1830,7 @@ end subroutine step_offline !! initializing the ocean state variables, and initializing subsidiary modules subroutine initialize_MOM(Time, Time_init, param_file, dirs, CS, restart_CSp, & Time_in, offline_tracer_mode, input_restart_file, diag_ptr, & - count_calls, tracer_flow_CSp, ice_shelf_CSp, waves_CSp) + count_calls, tracer_flow_CSp, ice_shelf_CSp, waves_CSp, ensemble_num) type(time_type), target, intent(inout) :: Time !< model time, set in this routine type(time_type), intent(in) :: Time_init !< The start time for the coupled model's calendar type(param_file_type), intent(out) :: param_file !< structure indicating parameter file to parse @@ -1853,7 +1853,9 @@ subroutine initialize_MOM(Time, Time_init, param_file, dirs, CS, restart_CSp, & !! dynamics timesteps. type(ice_shelf_CS), optional, pointer :: ice_shelf_CSp !< A pointer to an ice shelf control structure type(Wave_parameters_CS), & - optional, pointer :: Waves_CSp !< An optional pointer to a wave property CS + optional, pointer :: Waves_CSp !< An optional pointer to a wave property CS + integer, optional :: ensemble_num !< Ensemble index provided by the cap (instead of FMS + !! ensemble manager) ! local variables type(ocean_grid_type), pointer :: G => NULL() ! A pointer to the metric grid use for the run type(ocean_grid_type), pointer :: G_in => NULL() ! Pointer to the input grid @@ -1962,7 +1964,7 @@ subroutine initialize_MOM(Time, Time_init, param_file, dirs, CS, restart_CSp, & ! Read paths and filenames from namelist and store in "dirs". ! Also open the parsed input parameter file(s) and setup param_file. - call get_MOM_input(param_file, dirs, default_input_filename=input_restart_file) + call get_MOM_input(param_file, dirs, default_input_filename=input_restart_file, ensemble_num=ensemble_num) verbosity = 2 ; call read_param(param_file, "VERBOSITY", verbosity) call MOM_set_verbosity(verbosity) diff --git a/src/framework/MOM_file_parser.F90 b/src/framework/MOM_file_parser.F90 index 3a25981dc8..c85cccd9e2 100644 --- a/src/framework/MOM_file_parser.F90 +++ b/src/framework/MOM_file_parser.F90 @@ -124,7 +124,7 @@ module MOM_file_parser contains !> Make the contents of a parameter input file availalble in a param_file_type -subroutine open_param_file(filename, CS, checkable, component, doc_file_dir) +subroutine open_param_file(filename, CS, checkable, component, doc_file_dir, ensemble_num) character(len=*), intent(in) :: filename !< An input file name, optionally with the full path type(param_file_type), intent(inout) :: CS !< The control structure for the file_parser module, !! it is also a structure to parse for run-time parameters @@ -134,11 +134,13 @@ subroutine open_param_file(filename, CS, checkable, component, doc_file_dir) !! to generate parameter documentation file names; the default is"MOM" character(len=*), optional, intent(in) :: doc_file_dir !< An optional directory in which to write out !! the documentation files. The default is effectively './'. + integer, optional, intent(in) :: ensemble_num !< ensemble number to be appended to _doc filenames (optional) ! Local variables logical :: file_exists, Netcdf_file, may_check, reopened_file integer :: ios, iounit, strlen, i character(len=240) :: doc_path + character(len=5) :: ensemble_suffix type(parameter_block), pointer :: block => NULL() may_check = .true. ; if (present(checkable)) may_check = checkable @@ -211,6 +213,11 @@ subroutine open_param_file(filename, CS, checkable, component, doc_file_dir) call read_param(CS,"REPORT_UNUSED_PARAMS",CS%report_unused) call read_param(CS,"FATAL_UNUSED_PARAMS",CS%unused_params_fatal) CS%doc_file = "MOM_parameter_doc" + if (present(ensemble_num)) then + ! append instance suffix to doc_file + write(ensemble_suffix,'(A,I0.4)') '_', ensemble_num + CS%doc_file = trim(CS%doc_file)//ensemble_suffix + endif if (present(component)) CS%doc_file = trim(component)//"_parameter_doc" call read_param(CS,"DOCUMENT_FILE", CS%doc_file) if (.not.may_check) then diff --git a/src/framework/MOM_get_input.F90 b/src/framework/MOM_get_input.F90 index b6b5b89be9..4c643a5442 100644 --- a/src/framework/MOM_get_input.F90 +++ b/src/framework/MOM_get_input.F90 @@ -102,7 +102,7 @@ subroutine get_MOM_input(param_file, dirs, check_params, default_input_filename, if (len_trim(trim(parameter_filename(io))) > 0) then if (present(ensemble_num)) then call open_param_file(ensembler(parameter_filename(io),ensemble_num), param_file, & - check_params, doc_file_dir=output_dir) + check_params, doc_file_dir=output_dir, ensemble_num=ensemble_num) else call open_param_file(ensembler(parameter_filename(io)), param_file, & check_params, doc_file_dir=output_dir)