Accommodate multi-instance runs in CESM (#241)

* changes in nuopc cap, infra, and MOM.F90 to receive ensembe id from the coupler (alternative to FMS ensemble mngr)

* multi-instance logfile name correction in nuopc cap

* append ensemble suffix to _doc files

* changes in rpointer and restart file name handling to accommodate multi-instance CESM runs

* remove fms2_io_mod usage in FMS1/MOM_ensemble_manager_infra.F90

* rm whitespace in mom_cap

config_src/drivers/nuopc_cap/mom_cap.F90

@@ -28,9 +28,11 @@ module MOM_cap_mod
 use MOM_cap_methods,          only: mom_import, mom_export, mom_set_geomtype, mod2med_areacor
 use MOM_cap_methods,          only: med2mod_areacor, state_diagnose
 use MOM_cap_methods,          only: ChkErr
+use MOM_ensemble_manager,     only: ensemble_manager_init
 
 #ifdef CESMCOUPLED
 use shr_log_mod,             only: shr_log_setLogUnit
+use nuopc_shr_methods,       only: get_component_instance
 #endif
 use time_utils_mod,           only: esmf2fms_time
 
@@ -146,7 +148,8 @@ module MOM_cap_mod
 logical :: cesm_coupled = .false.
 type(ESMF_GeomType_Flag) :: geomtype
 #endif
-character(len=8) :: restart_mode = 'alarms'
+character(len=8)  :: restart_mode = 'alarms'
+character(len=16) :: inst_suffix = ''
 
 contains
 
@@ -422,6 +425,8 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
                                                                  ! (same as restartfile if single restart file)
   character(len=*), parameter            :: subname='(MOM_cap:InitializeAdvertise)'
   character(len=32)                      :: calendar
+  character(len=:), allocatable          :: rpointer_filename
+  integer                                :: inst_index
 !--------------------------------
 
   rc = ESMF_SUCCESS
@@ -451,6 +456,13 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
   CALL ESMF_TimeIntervalGet(TINT, S=DT_OCEAN, RC=rc)
   if (ChkErr(rc,__LINE__,u_FILE_u)) return
 
+#ifdef CESMCOUPLED
+  call get_component_instance(gcomp, inst_suffix, inst_index, rc)
+  if (ChkErr(rc,__LINE__,u_FILE_u)) return
+  call ensemble_manager_init(inst_suffix)
+  rpointer_filename = 'rpointer.ocn'//trim(inst_suffix)
+#endif
+
   ! reset shr logging to my log file
   if (localPet==0) then
     call NUOPC_CompAttributeGet(gcomp, name="diro", &
@@ -460,11 +472,19 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
          isPresent=isPresentLogfile, rc=rc)
     if (ChkErr(rc,__LINE__,u_FILE_u)) return
     if (isPresentDiro .and. isPresentLogfile) then
-         call NUOPC_CompAttributeGet(gcomp, name="diro", value=diro, rc=rc)
-         if (ChkErr(rc,__LINE__,u_FILE_u)) return
-         call NUOPC_CompAttributeGet(gcomp, name="logfile", value=logfile, rc=rc)
-         if (ChkErr(rc,__LINE__,u_FILE_u)) return
-         open(newunit=stdout,file=trim(diro)//"/"//trim(logfile))
+      call NUOPC_CompAttributeGet(gcomp, name="diro", value=diro, rc=rc)
+      if (ChkErr(rc,__LINE__,u_FILE_u)) return
+      call NUOPC_CompAttributeGet(gcomp, name="logfile", value=logfile, rc=rc)
+      if (ChkErr(rc,__LINE__,u_FILE_u)) return
+
+      if (cesm_coupled) then
+        ! Multiinstance logfile name needs a correction
+        if(logfile(4:4) == '_') then
+          logfile = logfile(1:3)//trim(inst_suffix)//logfile(9:)
+        endif
+      endif
+
+      open(newunit=stdout,file=trim(diro)//"/"//trim(logfile))
     else
       stdout = output_unit
     endif
@@ -521,12 +541,6 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
 
   time0 = set_date (YEAR,MONTH,DAY,HOUR,MINUTE,SECOND)
 
-
-  ! rsd need to figure out how to get this without share code
-  !call shr_nuopc_get_component_instance(gcomp, inst_suffix, inst_index)
-  !inst_name = "OCN"//trim(inst_suffix)
-
-
   if (is_root_pe()) then
     write(stdout,*) subname//'start time: y,m,d-',year,month,day,'h,m,s=',hour,minute,second
   endif
@@ -581,9 +595,9 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
 
       if (localPet == 0) then
         ! this hard coded for rpointer.ocn right now
-        open(newunit=readunit, file='rpointer.ocn', form='formatted', status='old', iostat=iostat)
+        open(newunit=readunit, file=rpointer_filename, form='formatted', status='old', iostat=iostat)
         if (iostat /= 0) then
-          call ESMF_LogSetError(ESMF_RC_FILE_OPEN, msg=subname//' ERROR opening rpointer.ocn', &
+          call ESMF_LogSetError(ESMF_RC_FILE_OPEN, msg=subname//' ERROR opening '//rpointer_filename, &
                  line=__LINE__, file=u_FILE_u, rcToReturn=rc)
           return
         endif
@@ -593,7 +607,7 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
             if (len(trim(restartfiles))>1 .and. iostat<0) then
               exit ! done reading restart files list.
             else
-              call ESMF_LogSetError(ESMF_RC_FILE_READ, msg=subname//' ERROR reading rpointer.ocn', &
+              call ESMF_LogSetError(ESMF_RC_FILE_READ, msg=subname//' ERROR reading '//rpointer_filename, &
                  line=__LINE__, file=u_FILE_u, rcToReturn=rc)
               return
             endif
@@ -616,7 +630,12 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
   endif
 
   ocean_public%is_ocean_pe = .true.
-  call ocean_model_init(ocean_public, ocean_state, time0, time_start, input_restart_file=trim(adjustl(restartfiles)))
+  if (cesm_coupled .and. len_trim(inst_suffix)>0) then
+    call ocean_model_init(ocean_public, ocean_state, time0, time_start, &
+      input_restart_file=trim(adjustl(restartfiles)), inst_index=inst_index)
+  else
+    call ocean_model_init(ocean_public, ocean_state, time0, time_start, input_restart_file=trim(adjustl(restartfiles)))
+  endif
 
   ! GMM, this call is not needed in CESM. Check with EMC if it can be deleted.
   call ocean_model_flux_init(ocean_state)
@@ -1489,6 +1508,7 @@ subroutine ModelAdvance(gcomp, rc)
   character(len=128)                     :: fldname
   character(len=*),parameter             :: subname='(MOM_cap:ModelAdvance)'
   character(len=8)                       :: suffix
+  character(len=:), allocatable          :: rpointer_filename
   integer                                :: num_rest_files
 
   rc = ESMF_SUCCESS
@@ -1658,20 +1678,26 @@ subroutine ModelAdvance(gcomp, rc)
         call ESMF_VMGet(vm, localPet=localPet, rc=rc)
         if (ChkErr(rc,__LINE__,u_FILE_u)) return
 
+        rpointer_filename = 'rpointer.ocn'//trim(inst_suffix)
+
         write(restartname,'(A,".mom6.r.",I4.4,"-",I2.2,"-",I2.2,"-",I5.5)') &
              trim(casename), year, month, day, seconds
         call ESMF_LogWrite("MOM_cap: Writing restart :  "//trim(restartname), ESMF_LOGMSG_INFO)
         ! write restart file(s)
         call ocean_model_restart(ocean_state, restartname=restartname, num_rest_files=num_rest_files)
         if (localPet == 0) then
            ! Write name of restart file in the rpointer file - this is currently hard-coded for the ocean
-          open(newunit=writeunit, file='rpointer.ocn', form='formatted', status='unknown', iostat=iostat)
+          open(newunit=writeunit, file=rpointer_filename, form='formatted', status='unknown', iostat=iostat)
           if (iostat /= 0) then
             call ESMF_LogSetError(ESMF_RC_FILE_OPEN, &
-                 msg=subname//' ERROR opening rpointer.ocn', line=__LINE__, file=u_FILE_u, rcToReturn=rc)
+                 msg=subname//' ERROR opening '//rpointer_filename, line=__LINE__, file=u_FILE_u, rcToReturn=rc)
             return
           endif
-          write(writeunit,'(a)') trim(restartname)//'.nc'
+          if (len_trim(inst_suffix) == 0) then
+            write(writeunit,'(a)') trim(restartname)//'.nc'
+          else
+            write(writeunit,'(a)') trim(restartname)//'.'//trim(inst_suffix)//'.nc'
+          endif
 
           if (num_rest_files > 1) then
             ! append i.th restart file name to rpointer

config_src/drivers/nuopc_cap/mom_ocean_model_nuopc.F90

@@ -231,7 +231,7 @@ module MOM_ocean_model_nuopc
 !!   This subroutine initializes both the ocean state and the ocean surface type.
 !! Because of the way that indicies and domains are handled, Ocean_sfc must have
 !! been used in a previous call to initialize_ocean_type.
-subroutine ocean_model_init(Ocean_sfc, OS, Time_init, Time_in, gas_fields_ocn, input_restart_file)
+subroutine ocean_model_init(Ocean_sfc, OS, Time_init, Time_in, gas_fields_ocn, input_restart_file, inst_index)
   type(ocean_public_type), target, &
                        intent(inout) :: Ocean_sfc !< A structure containing various publicly
                                 !! visible ocean surface properties after initialization,
@@ -248,6 +248,7 @@ subroutine ocean_model_init(Ocean_sfc, OS, Time_init, Time_in, gas_fields_ocn, i
                                               !! tracer fluxes, and can be used to spawn related
                                               !! internal variables in the ice model.
   character(len=*), optional, intent(in) :: input_restart_file !< If present, name of restart file to read
+  integer, optional :: inst_index !< Ensemble index provided by the cap (instead of FMS ensemble manager)
 
   ! Local variables
   real :: Rho0        ! The Boussinesq ocean density, in kg m-3.
@@ -285,7 +286,7 @@ subroutine ocean_model_init(Ocean_sfc, OS, Time_init, Time_in, gas_fields_ocn, i
                       OS%restart_CSp, Time_in, offline_tracer_mode=OS%offline_tracer_mode, &
                       input_restart_file=input_restart_file, &
                       diag_ptr=OS%diag, count_calls=.true., tracer_flow_CSp=OS%tracer_flow_CSp, &
-                      waves_CSp=OS%Waves)
+                      waves_CSp=OS%Waves, ensemble_num=inst_index)
   call get_MOM_state_elements(OS%MOM_CSp, G=OS%grid, GV=OS%GV, US=OS%US, C_p=OS%C_p, &
                               C_p_scaled=OS%fluxes%C_p, use_temp=use_temperature)
 

config_src/infra/FMS1/MOM_ensemble_manager_infra.F90

@@ -9,6 +9,7 @@ module MOM_ensemble_manager_infra
 use ensemble_manager_mod, only : FMS_get_ensemble_size => get_ensemble_size
 use ensemble_manager_mod, only : FMS_get_ensemble_pelist => get_ensemble_pelist
 use ensemble_manager_mod, only : FMS_get_ensemble_filter_pelist => get_ensemble_filter_pelist
+use fms_io_mod, only  : fms_io_set_filename_appendix=>set_filename_appendix
 
 implicit none ; private
 
@@ -20,9 +21,15 @@ module MOM_ensemble_manager_infra
 
 !> Initializes the ensemble manager which divides available resources
 !! in order to concurrently execute an ensemble of model realizations.
-subroutine ensemble_manager_init()
-
-  call FMS_ensemble_manager_init()
+subroutine ensemble_manager_init(ensemble_suffix)
+  character(len=*), optional, intent(in) :: ensemble_suffix !> Ensemble suffix provided by the cap. This may be
+                                                            !! provided to bypass FMS ensemble manager.
+
+  if (present(ensemble_suffix)) then
+    call fms_io_set_filename_appendix(trim(ensemble_suffix))
+  else
+    call FMS_ensemble_manager_init()
+  endif
 
 end subroutine ensemble_manager_init
 

config_src/infra/FMS2/MOM_ensemble_manager_infra.F90

@@ -9,6 +9,8 @@ module MOM_ensemble_manager_infra
 use ensemble_manager_mod, only : FMS_get_ensemble_size => get_ensemble_size
 use ensemble_manager_mod, only : FMS_get_ensemble_pelist => get_ensemble_pelist
 use ensemble_manager_mod, only : FMS_get_ensemble_filter_pelist => get_ensemble_filter_pelist
+use fms2_io_mod, only : fms2_io_set_filename_appendix=>set_filename_appendix
+use fms_io_mod, only  : fms_io_set_filename_appendix=>set_filename_appendix
 
 implicit none ; private
 
@@ -20,9 +22,16 @@ module MOM_ensemble_manager_infra
 
 !> Initializes the ensemble manager which divides available resources
 !! in order to concurrently execute an ensemble of model realizations.
-subroutine ensemble_manager_init()
-
-  call FMS_ensemble_manager_init()
+subroutine ensemble_manager_init(ensemble_suffix)
+  character(len=*), optional, intent(in) :: ensemble_suffix !> Ensemble suffix provided by the cap. This may be
+                                                            !! provided to bypass FMS ensemble manager.
+
+  if (present(ensemble_suffix)) then
+    call fms2_io_set_filename_appendix(trim(ensemble_suffix))
+    call fms_io_set_filename_appendix(trim(ensemble_suffix))
+  else
+    call FMS_ensemble_manager_init()
+  endif
 
 end subroutine ensemble_manager_init
 

src/core/MOM.F90

@@ -1830,7 +1830,7 @@ end subroutine step_offline
 !! initializing the ocean state variables, and initializing subsidiary modules
 subroutine initialize_MOM(Time, Time_init, param_file, dirs, CS, restart_CSp, &
                           Time_in, offline_tracer_mode, input_restart_file, diag_ptr, &
-                          count_calls, tracer_flow_CSp,  ice_shelf_CSp, waves_CSp)
+                          count_calls, tracer_flow_CSp,  ice_shelf_CSp, waves_CSp, ensemble_num)
   type(time_type), target,   intent(inout) :: Time        !< model time, set in this routine
   type(time_type),           intent(in)    :: Time_init   !< The start time for the coupled model's calendar
   type(param_file_type),     intent(out)   :: param_file  !< structure indicating parameter file to parse
@@ -1853,7 +1853,9 @@ subroutine initialize_MOM(Time, Time_init, param_file, dirs, CS, restart_CSp, &
                                                           !! dynamics timesteps.
   type(ice_shelf_CS), optional,     pointer :: ice_shelf_CSp !< A pointer to an ice shelf control structure
   type(Wave_parameters_CS), &
-                   optional, pointer       :: Waves_CSp       !< An optional pointer to a wave property CS
+                   optional, pointer       :: Waves_CSp   !< An optional pointer to a wave property CS
+  integer, optional :: ensemble_num                       !< Ensemble index provided by the cap (instead of FMS
+                                                          !! ensemble manager)
   ! local variables
   type(ocean_grid_type),  pointer :: G => NULL()    ! A pointer to the metric grid use for the run
   type(ocean_grid_type),  pointer :: G_in => NULL() ! Pointer to the input grid
@@ -1962,7 +1964,7 @@ subroutine initialize_MOM(Time, Time_init, param_file, dirs, CS, restart_CSp, &
 
   ! Read paths and filenames from namelist and store in "dirs".
   ! Also open the parsed input parameter file(s) and setup param_file.
-  call get_MOM_input(param_file, dirs, default_input_filename=input_restart_file)
+  call get_MOM_input(param_file, dirs, default_input_filename=input_restart_file, ensemble_num=ensemble_num)
 
   verbosity = 2 ; call read_param(param_file, "VERBOSITY", verbosity)
   call MOM_set_verbosity(verbosity)

src/framework/MOM_file_parser.F90

@@ -124,7 +124,7 @@ module MOM_file_parser
 contains
 
 !> Make the contents of a parameter input file availalble in a param_file_type
-subroutine open_param_file(filename, CS, checkable, component, doc_file_dir)
+subroutine open_param_file(filename, CS, checkable, component, doc_file_dir, ensemble_num)
   character(len=*),           intent(in) :: filename !< An input file name, optionally with the full path
   type(param_file_type),   intent(inout) :: CS      !< The control structure for the file_parser module,
                                          !! it is also a structure to parse for run-time parameters
@@ -134,11 +134,13 @@ subroutine open_param_file(filename, CS, checkable, component, doc_file_dir)
                                          !! to generate parameter documentation file names; the default is"MOM"
   character(len=*), optional, intent(in) :: doc_file_dir !< An optional directory in which to write out
                                          !! the documentation files.  The default is effectively './'.
+  integer, optional, intent(in)          :: ensemble_num !< ensemble number to be appended to _doc filenames (optional)
 
   ! Local variables
   logical :: file_exists, Netcdf_file, may_check, reopened_file
   integer :: ios, iounit, strlen, i
   character(len=240) :: doc_path
+  character(len=5)  :: ensemble_suffix
   type(parameter_block), pointer :: block => NULL()
 
   may_check = .true. ; if (present(checkable)) may_check = checkable
@@ -211,6 +213,11 @@ subroutine open_param_file(filename, CS, checkable, component, doc_file_dir)
   call read_param(CS,"REPORT_UNUSED_PARAMS",CS%report_unused)
   call read_param(CS,"FATAL_UNUSED_PARAMS",CS%unused_params_fatal)
   CS%doc_file = "MOM_parameter_doc"
+  if (present(ensemble_num)) then
+    ! append instance suffix to doc_file
+    write(ensemble_suffix,'(A,I0.4)') '_', ensemble_num
+    CS%doc_file = trim(CS%doc_file)//ensemble_suffix
+  endif
   if (present(component)) CS%doc_file = trim(component)//"_parameter_doc"
   call read_param(CS,"DOCUMENT_FILE", CS%doc_file)
   if (.not.may_check) then

src/framework/MOM_get_input.F90

@@ -102,7 +102,7 @@ subroutine get_MOM_input(param_file, dirs, check_params, default_input_filename,
       if (len_trim(trim(parameter_filename(io))) > 0) then
         if (present(ensemble_num)) then
           call open_param_file(ensembler(parameter_filename(io),ensemble_num), param_file, &
-               check_params, doc_file_dir=output_dir)
+               check_params, doc_file_dir=output_dir, ensemble_num=ensemble_num)
         else
           call open_param_file(ensembler(parameter_filename(io)), param_file, &
                check_params, doc_file_dir=output_dir)