Merge pull request #337 from jedwards4b/esmf_multidriver

redo multiinstance support

cesm/driver/ensemble_driver.F90

@@ -79,17 +79,17 @@ subroutine SetModelServices(ensemble_driver, rc)
     use ESMF          , only : ESMF_CalendarSetDefault
     use ESMF          , only : ESMF_CALKIND_NOLEAP, ESMF_CALKIND_GREGORIAN
     use NUOPC         , only : NUOPC_CompAttributeGet, NUOPC_CompAttributeSet, NUOPC_CompAttributeAdd
-    use NUOPC_Driver  , only : NUOPC_DriverAddComp
+    use NUOPC_Driver  , only : NUOPC_DriverAddComp, NUOPC_DriverGetComp
     use esm           , only : ESMSetServices => SetServices, ReadAttributes
     use esm_time_mod  , only : esm_time_clockInit
-
+    use perf_mod      , only : t_startf, t_stopf, t_initf
     ! input/output variables
     type(ESMF_GridComp)    :: ensemble_driver
     integer, intent(out)   :: rc
 
     ! local variables
     type(ESMF_VM)          :: vm
-    type(ESMF_GridComp)    :: driver, gridcomptmp
+    type(ESMF_GridComp)    :: driver
     type(ESMF_Config)      :: config
     integer                :: n
     integer, pointer       :: petList(:)
@@ -102,6 +102,7 @@ subroutine SetModelServices(ensemble_driver, rc)
     integer                :: inst
     integer                :: number_of_members
     integer                :: ntasks_per_member
+    integer                :: Global_Comm
     character(CL)          :: start_type     ! Type of startup
     character(len=7)       :: drvrinst
     character(len=5)       :: inst_suffix
@@ -116,10 +117,21 @@ subroutine SetModelServices(ensemble_driver, rc)
 
     rc = ESMF_SUCCESS
     call ESMF_LogWrite(trim(subname)//": called", ESMF_LOGMSG_INFO)
-
     call ESMF_GridCompGet(ensemble_driver, config=config, vm=vm, rc=rc)
     if (chkerr(rc,__LINE__,u_FILE_u)) return
 
+    call ESMF_VMGet(vm, localPet=localPet, mpiCommunicator=global_comm, rc=rc)
+    if (chkerr(rc,__LINE__,u_FILE_u)) return
+
+    if (localPet == 0) then
+       maintask=.true.
+    else
+       maintask = .false.
+    end if
+
+    call t_initf('drv_in', LogPrint=.true., LogUnit=logunit, mpicom=global_comm, mastertask=maintask)
+    call t_startf(subname)
+
     !-------------------------------------------
     ! Initialize clocks
     !-------------------------------------------
@@ -200,23 +212,32 @@ subroutine SetModelServices(ensemble_driver, rc)
     !-------------------------------------------
 
     allocate(petList(ntasks_per_member))
-    ! which driver instance is this?
-    inst = localPet/ntasks_per_member + 1
+    ! We need to loop over instances
+    call t_startf('compute_drivers')
+    do inst = 1, number_of_members
+
+       ! Determine pet list for driver instance
+       petList(1) = (inst-1) * ntasks_per_member
+       do n=2,ntasks_per_member
+          petList(n) = petList(n-1) + 1
+       enddo
+
+       ! Add driver instance to ensemble driver
+       write(drvrinst,'(a,i4.4)') "ESM",inst
+       call NUOPC_DriverAddComp(ensemble_driver, drvrinst, ESMSetServices, petList=petList, rc=rc)
+       if (chkerr(rc,__LINE__,u_FILE_u)) return
+    enddo
+    call t_stopf('compute_drivers')
 
-    ! Determine pet list for driver instance
+    inst = localPet/ntasks_per_member + 1
     petList(1) = (inst-1) * ntasks_per_member
     do n=2,ntasks_per_member
        petList(n) = petList(n-1) + 1
     enddo
-
-    ! Add driver instance to ensemble driver
-    write(drvrinst,'(a,i4.4)') "ESM",inst
-    call NUOPC_DriverAddComp(ensemble_driver, drvrinst, ESMSetServices, petList=petList, comp=gridcomptmp, rc=rc)
-    if (chkerr(rc,__LINE__,u_FILE_u)) return
-
     if (localpet >= petlist(1) .and. localpet <= petlist(ntasks_per_member)) then
-
-       driver = gridcomptmp
+       write(drvrinst,'(a,i4.4)') "ESM",inst
+       call NUOPC_DriverGetComp(ensemble_driver, drvrinst, comp=driver, rc=rc)
+       if (chkerr(rc,__LINE__,u_FILE_u)) return
 
        if(number_of_members > 1) then
           call NUOPC_CompAttributeAdd(driver, attrList=(/'inst_suffix'/), rc=rc)
@@ -264,6 +285,7 @@ subroutine SetModelServices(ensemble_driver, rc)
     endif
 
     deallocate(petList)
+    call t_stopf(subname)
 
     call ESMF_LogWrite(trim(subname)//": done", ESMF_LOGMSG_INFO)
 

cesm/driver/esm.F90

@@ -10,7 +10,6 @@ module ESM
   use shr_mem_mod  , only : shr_mem_init
   use shr_log_mod  , only : shr_log_setLogunit
   use esm_utils_mod, only : logunit, maintask, dbug_flag, chkerr
-  use perf_mod     , only : t_initf, t_setLogUnit
 
   implicit none
   private
@@ -151,8 +150,6 @@ subroutine SetModelServices(driver, rc)
     call ESMF_VMGet(vm, localPet=localPet, mpiCommunicator=global_comm, rc=rc)
     if (chkerr(rc,__LINE__,u_FILE_u)) return
 
-    call ESMF_VMGet(vm, localPet=localPet, rc=rc)
-    if (chkerr(rc,__LINE__,u_FILE_u)) return
     if (localPet == 0) then
        maintask=.true.
     else
@@ -211,11 +208,6 @@ subroutine SetModelServices(driver, rc)
        write(logunit,*) trim(meminitstr)
     end if
 
-    !-------------------------------------------
-    ! Timer initialization (has to be after pelayouts are determined)
-    !-------------------------------------------
-    call t_initf('drv_in', LogPrint=.true., LogUnit=logunit, mpicom=global_comm, mastertask=maintask, MaxThreads=maxthreads)
-
     call ESMF_LogWrite(trim(subname)//": done", ESMF_LOGMSG_INFO)
 
   end subroutine SetModelServices
@@ -636,7 +628,7 @@ subroutine AddAttributes(gcomp, driver, config, compid, compname, inst_suffix, n
     if (chkerr(rc,__LINE__,u_FILE_u)) return
 
     !------
-    ! Add driver restart flag a to gcomp attributes
+    ! Add driver restart flag to gcomp attributes
     !------
     attribute = 'read_restart'
     call NUOPC_CompAttributeGet(driver, name=trim(attribute), value=cvalue, rc=rc)

cesm/nuopc_cap_share/nuopc_shr_methods.F90

@@ -145,6 +145,7 @@ subroutine set_component_logging(gcomp, maintask, logunit, shrlogunit, rc)
     character(len=CL) :: logfile
     character(len=CL) :: inst_suffix
     integer :: inst_index  ! not used here
+    integer :: n
     !-----------------------------------------------------------------------
 
     rc = ESMF_SUCCESS
@@ -157,8 +158,9 @@ subroutine set_component_logging(gcomp, maintask, logunit, shrlogunit, rc)
        call get_component_instance(gcomp, inst_suffix, inst_index, rc=rc)
        if (chkerr(rc,__LINE__,u_FILE_u)) return
        ! Multiinstance logfile name needs a correction
-       if(logfile(4:4) == '_') then
-          logfile = logfile(1:3)//trim(inst_suffix)//logfile(9:)
+       if(len_trim(inst_suffix) > 0) then
+          n = index(logfile, '.')
+          logfile = logfile(1:n-1)//trim(inst_suffix)//logfile(n:)
        endif
 
        open(newunit=logunit,file=trim(diro)//"/"//trim(logfile))