Skip to content

Commit

Permalink
Merge pull request #1 from CMCC-Foundation/dev/cmcc-cm3
Browse files Browse the repository at this point in the history 
Dev/cmcc cm3
Update to cmeps0.13.81 for CMCC-CM3
  • Loading branch information
@daniele-peano
daniele-peano authored Dec 15, 2022
2 parents c03f5a4 + ae5cfcd commit 9e51b44
Show file tree
Hide file tree
Showing 59 changed files with 8,990 additions and 3,839 deletions.
31 changes: 20 additions & 11 deletions .github/workflows/extbuild.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,10 @@ jobs:
CXX: mpicxx
CPPFLAGS: "-I/usr/include -I/usr/local/include"
# Versions of all dependencies can be updated here
ESMF_VERSION: ESMF_8_2_0_beta_snapshot_14
PNETCDF_VERSION: pnetcdf-1.12.2
ESMF_VERSION: v8.3.0b13
PNETCDF_VERSION: pnetcdf-1.12.3
NETCDF_FORTRAN_VERSION: v4.5.2
# PIO version is awkward
PIO_VERSION_DIR: pio2_5_3
PIO_VERSION: pio-2.5.3
PIO_VERSION: pio2_5_7
steps:
- uses: actions/checkout@v2
# Build the ESMF library, if the cache contains a previous build
Expand All @@ -38,12 +36,19 @@ jobs:
run: |
sudo apt-get update
sudo apt-get install gfortran wget openmpi-bin netcdf-bin libopenmpi-dev libnetcdf-dev
- id: checkout-ESMF
uses: actions/checkout@v3
with:
repository: esmf-org/esmf
path: esmf-src
ref: ${{ env.ESMF_VERSION }}
- id: build-ESMF
if: steps.cache-esmf.outputs.cache-hit != 'true'
run: |
wget https://github.com/esmf-org/esmf/archive/${{ env.ESMF_VERSION }}.tar.gz
tar -xzvf ${{ env.ESMF_VERSION }}.tar.gz
pushd esmf-${{ env.ESMF_VERSION }}
#wget https://github.com/esmf-org/esmf/archive/${{ env.ESMF_VERSION }}.tar.gz
#tar -xzvf ${{ env.ESMF_VERSION }}.tar.gz
#pushd esmf-${{ env.ESMF_VERSION }}
pushd esmf-src
export ESMF_DIR=`pwd`
export ESMF_COMM=openmpi
export ESMF_YAMLCPP="internal"
Expand Down Expand Up @@ -95,14 +100,18 @@ jobs:
${{ runner.os }}-${{ env.NETCDF_FORTRAN_VERSION }}-netcdf-fortran
${{ runner.os }}-${{ env.PNETCDF_VERSION }}-pnetcdf
- id: checkout-PIO
uses: actions/checkout@v3
with:
repository: NCAR/ParallelIO
path: parallelio-src
ref: ${{ env.PIO_VERSION }}
- name: Build PIO
if: steps.cache-PIO.outputs.cache-hit != 'true'
run: |
wget https://github.com/NCAR/ParallelIO/releases/download/${{ env.PIO_VERSION_DIR }}/${{ env.PIO_VERSION }}.tar.gz
tar -xzvf ${{ env.PIO_VERSION }}.tar.gz
mkdir build-pio
pushd build-pio
cmake -Wno-dev -DNetCDF_C_LIBRARY=/usr/lib/x86_64-linux-gnu/libnetcdf.so -DNetCDF_C_INCLUDE_DIR=/usr/include -DCMAKE_PREFIX_PATH=/usr -DCMAKE_INSTALL_PREFIX=$HOME/pio -DPIO_HDF5_LOGGING=On -DPIO_USE_MALLOC=On -DPIO_ENABLE_TESTS=Off -DPIO_ENABLE_LOGGING=On -DPIO_ENABLE_EXAMPLES=Off -DPIO_ENABLE_TIMING=Off -DNetCDF_Fortran_PATH=$HOME/netcdf-fortran -DPnetCDF_PATH=$HOME/pnetcdf ../${{ env.PIO_VERSION }}
cmake -Wno-dev -DNetCDF_C_LIBRARY=/usr/lib/x86_64-linux-gnu/libnetcdf.so -DNetCDF_C_INCLUDE_DIR=/usr/include -DCMAKE_PREFIX_PATH=/usr -DCMAKE_INSTALL_PREFIX=$HOME/pio -DPIO_HDF5_LOGGING=On -DPIO_USE_MALLOC=On -DPIO_ENABLE_TESTS=Off -DPIO_ENABLE_LOGGING=On -DPIO_ENABLE_EXAMPLES=Off -DPIO_ENABLE_TIMING=Off -DNetCDF_Fortran_PATH=$HOME/netcdf-fortran -DPnetCDF_PATH=$HOME/pnetcdf ../parallelio-src
make VERBOSE=1
make install
popd
Expand Down
2 changes: 1 addition & 1 deletion cesm/driver/ensemble_driver.F90
View file
Original file line number Diff line number Diff line change
Expand Up @@ -244,7 +244,7 @@ subroutine SetModelServices(ensemble_driver, rc)
call ReadAttributes(driver, config, "DRIVER_attributes::", rc=rc)
if (chkerr(rc,__LINE__,u_FILE_u)) return

call ReadAttributes(driver, config, "DRV_modelio"//trim(inst_suffix)//"::", rc=rc)
call ReadAttributes(driver, config, "DRV_modelio::", rc=rc)
if (chkerr(rc,__LINE__,u_FILE_u)) return

! Set the driver log to the driver task 0
Expand Down
104 changes: 79 additions & 25 deletions cesm/driver/esm.F90
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,12 @@ module ESM
!-----------------------------------------------------------------------------

use shr_kind_mod , only : r8=>shr_kind_r8, cl=>shr_kind_cl, cs=>shr_kind_cs
use shr_log_mod , only : shrlogunit=> shr_log_unit
use shr_sys_mod , only : shr_sys_abort
use shr_mpi_mod , only : shr_mpi_bcast
use shr_mem_mod , only : shr_mem_init
use shr_file_mod , only : shr_file_setLogunit
use esm_utils_mod, only : logunit, mastertask, dbug_flag, chkerr
use perf_mod , only : t_initf
use perf_mod , only : t_initf, t_setLogUnit

implicit none
private
Expand Down Expand Up @@ -220,8 +219,7 @@ subroutine SetModelServices(driver, rc)
!-------------------------------------------
! Timer initialization (has to be after pelayouts are determined)
!-------------------------------------------

call t_initf('drv_in', LogPrint=.true., mpicom=global_comm, mastertask=mastertask, MaxThreads=maxthreads)
call t_initf('drv_in', LogPrint=.true., LogUnit=logunit, mpicom=global_comm, mastertask=mastertask, MaxThreads=maxthreads)

call ESMF_LogWrite(trim(subname)//": done", ESMF_LOGMSG_INFO)

Expand Down Expand Up @@ -670,8 +668,11 @@ subroutine AddAttributes(gcomp, driver, config, compid, compname, inst_suffix, n
if (chkerr(rc,__LINE__,u_FILE_u)) return
call ReadAttributes(gcomp, config, "ALLCOMP_attributes::", rc=rc)
if (chkerr(rc,__LINE__,u_FILE_u)) return
call ReadAttributes(gcomp, config, trim(compname)//"_modelio"//trim(inst_suffix)//"::", rc=rc)
if (chkerr(rc,__LINE__,u_FILE_u)) return
call ReadAttributes(gcomp, config, trim(compname)//"_modelio::", rc=rc)
if (chkerr(rc,__LINE__,u_FILE_u)) then
print *,__FILE__,__LINE__,"ERROR reading ",trim(compname)," modelio from runconfig"
return
endif
call ReadAttributes(gcomp, config, "CLOCK_attributes::", rc=rc)
if (chkerr(rc,__LINE__,u_FILE_u)) return

Expand Down Expand Up @@ -807,7 +808,7 @@ subroutine esm_init_pelayout(driver, maxthreads, rc)
use mpi , only : MPI_COMM_NULL, mpi_comm_size
#endif
use mct_mod , only : mct_world_init
use shr_pio_mod , only : shr_pio_init2
use driver_pio_mod , only : driver_pio_init, driver_pio_component_init

#ifdef MED_PRESENT
use med_internalstate_mod , only : med_id
Expand Down Expand Up @@ -931,6 +932,11 @@ subroutine esm_init_pelayout(driver, maxthreads, rc)
inst_suffix = ""
endif

! Initialize PIO
! This reads in the pio parameters that are independent of component
call driver_pio_init(driver, rc=rc)
if (chkerr(rc,__LINE__,u_FILE_u)) return

allocate(comms(componentCount+1), comps(componentCount+1))
comps(1) = 1
comms = MPI_COMM_NULL
Expand Down Expand Up @@ -1174,12 +1180,14 @@ subroutine esm_init_pelayout(driver, maxthreads, rc)
if (chkerr(rc,__LINE__,u_FILE_u)) return

enddo
! Read in component dependent PIO parameters and initialize
! IO systems
call driver_pio_component_init(driver, size(comps), rc)
if (chkerr(rc,__LINE__,u_FILE_u)) return

! Initialize MCT (this is needed for data models and cice prescribed capability)
call mct_world_init(componentCount+1, GLOBAL_COMM, comms, comps)

! Initialize PIO
call shr_pio_init2(comps(2:), compLabels, comp_iamin, comms(2:), comp_comm_iam)

deallocate(petlist, comms, comps, comp_iamin, comp_comm_iam)

Expand All @@ -1195,6 +1203,8 @@ subroutine esm_set_single_column_attributes(compname, gcomp, rc)
use netcdf, only : nf90_inq_dimid, nf90_inquire_dimension, nf90_inq_varid, nf90_get_var
use NUOPC , only : NUOPC_CompAttributeGet, NUOPC_CompAttributeSet, NUOPC_CompAttributeAdd
use ESMF , only : ESMF_GridComp, ESMF_GridCompGet, ESMF_VM, ESMF_VMGet, ESMF_SUCCESS
use ESMF , only : ESMF_Mesh, ESMF_MeshCreate, ESMF_FILEFORMAT_ESMFMESH, ESMF_MeshGet, ESMF_MESHLOC_ELEMENT
use ESMF , only : ESMF_Field, ESMF_FieldCreate, ESMF_FieldGet, ESMF_FieldRegridGetArea, ESMF_TYPEKIND_r8

! input/output variables
character(len=*) , intent(in) :: compname
Expand All @@ -1204,14 +1214,24 @@ subroutine esm_set_single_column_attributes(compname, gcomp, rc)
! local variables
type(ESMF_VM) :: vm
character(len=CL) :: single_column_lnd_domainfile
character(len=CL) :: single_column_global_meshfile
real(r8) :: scol_lon
real(r8) :: scol_lat
real(r8) :: scol_area
integer :: scol_lndmask
real(r8) :: scol_lndfrac
integer :: scol_ocnmask
real(r8) :: scol_ocnfrac
integer :: i,j,ni,nj
integer :: scol_mesh_n
type(ESMF_Mesh) :: mesh
type(ESMF_Field) :: lfield
integer :: lsize
integer :: spatialdim
real(r8), pointer :: ownedElemCoords(:)
real(r8), pointer :: latMesh(:)
real(r8), pointer :: lonMesh(:)
real(r8), pointer :: dataptr(:)
integer :: i,j,ni,nj,n
integer :: ncid
integer :: dimid
integer :: varid_xc
Expand All @@ -1235,7 +1255,6 @@ subroutine esm_set_single_column_attributes(compname, gcomp, rc)
character(len=*), parameter :: subname= ' (esm_get_single_column_attributes) '
!-------------------------------------------------------------------------------


rc = ESMF_SUCCESS

! obtain the single column lon and lat
Expand All @@ -1247,6 +1266,8 @@ subroutine esm_set_single_column_attributes(compname, gcomp, rc)
read(cvalue,*) scol_lat
call NUOPC_CompAttributeGet(gcomp, name='single_column_lnd_domainfile', value=single_column_lnd_domainfile, rc=rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return
call NUOPC_CompAttributeGet(gcomp, name='mesh_atm', value=single_column_global_meshfile, rc=rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return
call NUOPC_CompAttributeAdd(gcomp, attrList=(/'scol_spval'/), rc=rc)
if (chkerr(rc,__LINE__,u_FILE_u)) return

Expand Down Expand Up @@ -1341,6 +1362,7 @@ subroutine esm_set_single_column_attributes(compname, gcomp, rc)
do j = 1,nj
lats(j) = glob_grid(1,j)
end do

! find nearest neighbor indices of scol_lon and scol_lat in single_column_lnd_domain file
! convert lons array and scol_lon to 0,360 and find index of value closest to 0
! and obtain single-column longitude/latitude indices to retrieve
Expand Down Expand Up @@ -1380,26 +1402,53 @@ subroutine esm_set_single_column_attributes(compname, gcomp, rc)
//' ocean and land mask cannot both be zero')
end if

status = nf90_close(ncid)
if (status /= nf90_noerr) call shr_sys_abort (trim(subname) //': closing '//&
trim(single_column_lnd_domainfile))

! Now read in mesh file to get exact values of scol_lon and scol_lat that will be used
! by the models - assume that this occurs only on 1 processor
mesh = ESMF_MeshCreate(filename=trim(single_column_global_meshfile), fileformat=ESMF_FILEFORMAT_ESMFMESH, rc=rc)
if (chkerr(rc,__LINE__,u_FILE_u)) return
call ESMF_MeshGet(mesh, spatialDim=spatialDim, numOwnedElements=lsize, rc=rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return
allocate(ownedElemCoords(spatialDim*lsize))
allocate(lonMesh(lsize), latMesh(lsize))
call ESMF_MeshGet(mesh, ownedElemCoords=ownedElemCoords)
if (ChkErr(rc,__LINE__,u_FILE_u)) return
do n = 1,lsize
lonMesh(n) = ownedElemCoords(2*n-1)
latMesh(n) = ownedElemCoords(2*n)
if (abs(lonMesh(n) - scol_lon) < 1.e-4 .and. abs(latMesh(n) - scol_lat) < 1.e-4) then
scol_mesh_n = n
scol_mesh_n = n
exit
end if
end do
scol_lon = lonMesh(scol_mesh_n)
scol_lat = latMesh(scol_mesh_n)

! Obtain mesh info areas
lfield = ESMF_FieldCreate(mesh, ESMF_TYPEKIND_r8, name='area', meshloc=ESMF_MESHLOC_ELEMENT, rc=rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return
call ESMF_FieldRegridGetArea(lfield, rc=rc)
if (chkerr(rc,__LINE__,u_FILE_u)) return
call ESMF_FieldGet(lfield, farrayPtr=dataptr, rc=rc)
if (chkerr(rc,__LINE__,u_FILE_u)) return
scol_area = dataptr(scol_mesh_n)

! Set single column attribute values for all components
write(cvalue,*) scol_lon
call NUOPC_CompAttributeSet(gcomp, name='scol_lon', value=trim(cvalue), rc=rc)
if (chkerr(rc,__LINE__,u_FILE_u)) return

write(cvalue,*) scol_lat
call NUOPC_CompAttributeSet(gcomp, name='scol_lat', value=trim(cvalue), rc=rc)
if (chkerr(rc,__LINE__,u_FILE_u)) return

write(cvalue,*) ni
call NUOPC_CompAttributeSet(gcomp, name='scol_ni', value=trim(cvalue), rc=rc)
write(cvalue,*) scol_area
call NUOPC_CompAttributeSet(gcomp, name='scol_area', value=trim(cvalue), rc=rc)
if (chkerr(rc,__LINE__,u_FILE_u)) return

write(cvalue,*) nj
call NUOPC_CompAttributeSet(gcomp, name='scol_nj', value=trim(cvalue), rc=rc)
if (chkerr(rc,__LINE__,u_FILE_u)) return

status = nf90_close(ncid)
if (status /= nf90_noerr) call shr_sys_abort (trim(subname) //': closing '//&
trim(single_column_lnd_domainfile))

! Write out diagnostic info
write(logunit,'(a,2(f13.5,2x))')trim(subname)//' nearest neighbor scol_lon and scol_lat in '&
//trim(single_column_lnd_domainfile)//' are ',scol_lon,scol_lat
if (trim(compname) == 'LND') then
Expand All @@ -1411,6 +1460,12 @@ subroutine esm_set_single_column_attributes(compname, gcomp, rc)
else
write(logunit,'(a)')trim(subname)//' atm point has unit mask and unit fraction '
end if
write(cvalue,*) ni
call NUOPC_CompAttributeSet(gcomp, name='scol_ni', value=trim(cvalue), rc=rc)
if (chkerr(rc,__LINE__,u_FILE_u)) return
write(cvalue,*) nj
call NUOPC_CompAttributeSet(gcomp, name='scol_nj', value=trim(cvalue), rc=rc)
if (chkerr(rc,__LINE__,u_FILE_u)) return

else

Expand All @@ -1423,12 +1478,11 @@ subroutine esm_set_single_column_attributes(compname, gcomp, rc)
scol_ocnfrac = 1._r8
scol_area = 1.e30

write(cvalue,*) 1
call NUOPC_CompAttributeSet(gcomp, name='scol_ni', value=trim(cvalue), rc=rc)
if (chkerr(rc,__LINE__,u_FILE_u)) return
write(cvalue,*) 1
call NUOPC_CompAttributeSet(gcomp, name='scol_nj', value=trim(cvalue), rc=rc)
if (chkerr(rc,__LINE__,u_FILE_u)) return
write(cvalue,*) 1

write(logunit,'(a)')' single point mode is active'
write(logunit,'(a,f13.5,a,f13.5,a)')' scol_lon is ',scol_lon,' and scol_lat is '
Expand Down
12 changes: 0 additions & 12 deletions cesm/driver/esmApp.F90
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ program esmApp
use mpi
use NUOPC, only : NUOPC_FieldDictionarySetup
use ensemble_driver, only : SetServices
use shr_pio_mod, only : shr_pio_init1
use shr_sys_mod, only : shr_sys_abort

implicit none
Expand Down Expand Up @@ -44,17 +43,6 @@ program esmApp
#endif
COMP_COMM = MPI_COMM_WORLD

!-----------------------------------------------------------------------------
! Initialize PIO
!-----------------------------------------------------------------------------

! For planned future use of async io using pio2. The IO tasks are seperated from the compute tasks here
! and COMP_COMM will be MPI_COMM_NULL on the IO tasks which then call shr_pio_init2 and do not return until
! the model completes. All other tasks call ESMF_Initialize. 8 is the maximum number of component models
! supported

call shr_pio_init1(8, "drv_in", COMP_COMM)

!-----------------------------------------------------------------------------
! Initialize ESMF
!-----------------------------------------------------------------------------
Expand Down
Loading

0 comments on commit 9e51b44

Please sign in to comment.