From fc5401da49473897d7ac1d0be1284bb01b17bcc3 Mon Sep 17 00:00:00 2001 From: Gerhard Theurich Date: Tue, 10 Dec 2024 11:47:48 -0800 Subject: [PATCH 1/3] Debugging log around RH creation. --- fv3_cap.F90 | 57 ++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 54 insertions(+), 3 deletions(-) diff --git a/fv3_cap.F90 b/fv3_cap.F90 index 4d3f999d2..b01b8b628 100644 --- a/fv3_cap.F90 +++ b/fv3_cap.F90 @@ -784,8 +784,9 @@ subroutine InitializeAdvertise(gcomp, rc) endif endif - call ESMF_LogWrite('bf FieldBundleRegridStore', ESMF_LOGMSG_INFO, rc=rc) - write(msgString,"(A,I2.2,',',I2.2,A)") "calling into wrtFB(",j,i, ") FieldBundleRegridStore()...." + call ESMF_LogSet(flush=.true., rc=rc) + + write(msgString,"(A,I2.2,',',I2.2,A)") "RH creation for wrtFB(",j,i, ") ...." call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) if (i==1) then @@ -795,12 +796,27 @@ subroutine InitializeAdvertise(gcomp, rc) if (rh_file_exist .and. use_saved_routehandles) then if(mype==0) print *,'in fv3cap init, routehandle file ',trim(rh_filename), ' exists' + + call ESMF_VMBarrier(VM, rc=rc) + write(msgString,*) "Calling into ESMF_RouteHandleCreate(from file)...", trim(rh_filename) + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + routehandle(j,1) = ESMF_RouteHandleCreate(fileName=trim(rh_filename), rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + + call ESMF_VMBarrier(VM, rc=rc) + write(msgString,*) "... returned from ESMF_RouteHandleCreate(from file)." + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + else ! this is a Store() for the first wrtComp -> must do the Store() if (fieldbundle_uses_redist) then call ESMF_TraceRegionEnter("ESMF_FieldBundleRedistStore()", rc=rc) + + call ESMF_VMBarrier(VM, rc=rc) + write(msgString,*) "Calling into FieldBundleRedistStore..." + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + call ESMF_FieldBundleRedistStore(fcstFB(j), wrtFB(j,1), & routehandle=routehandle(j,1), & rc=rc) @@ -809,11 +825,21 @@ subroutine InitializeAdvertise(gcomp, rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return ! call ESMF_Finalize(endflag=ESMF_END_ABORT) endif + + call ESMF_VMBarrier(VM, rc=rc) + write(msgString,*) "... returned from FieldBundleRedistStore." + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + call ESMF_TraceRegionExit("ESMF_FieldBundleRedistStore()", rc=rc) call ESMF_LogWrite('af FieldBundleRedistStore', ESMF_LOGMSG_INFO, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return else call ESMF_TraceRegionEnter("ESMF_FieldBundleRegridStore()", rc=rc) + + call ESMF_VMBarrier(VM, rc=rc) + write(msgString,*) "Calling into FieldBundleRegridStore..." + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + call ESMF_FieldBundleRegridStore(fcstFB(j), wrtFB(j,1), & regridMethod=regridmethod, routehandle=routehandle(j,1), & unmappedaction=ESMF_UNMAPPEDACTION_IGNORE, & @@ -822,15 +848,30 @@ subroutine InitializeAdvertise(gcomp, rc) call ESMF_LogWrite('fv3_cap.F90:InitializeAdvertise error in ESMF_FieldBundleRegridStore', ESMF_LOGMSG_ERROR, rc=rc) call ESMF_Finalize(endflag=ESMF_END_ABORT) endif + + call ESMF_VMBarrier(VM, rc=rc) + write(msgString,*) "... returned from FieldBundleRegridStore." + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + call ESMF_TraceRegionExit("ESMF_FieldBundleRegridStore()", rc=rc) call ESMF_LogWrite('af FieldBundleRegridStore', ESMF_LOGMSG_INFO, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return endif if (use_saved_routehandles) then + + call ESMF_VMBarrier(VM, rc=rc) + write(msgString,*) "Calling into ESMF_RouteHandleWrite..." + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + call ESMF_RouteHandleWrite(routehandle(j,1), fileName=trim(rh_filename), rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return if(mype==0) print *,'in fv3cap init, saved routehandle file ',trim(rh_filename) + + call ESMF_VMBarrier(VM, rc=rc) + write(msgString,*) "... returned from ESMF_RouteHandleWrite." + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + endif endif @@ -842,14 +883,24 @@ subroutine InitializeAdvertise(gcomp, rc) targetPetList(1:num_pes_fcst) = fcstPetList(:) targetPetList(num_pes_fcst+1:) = petList(:) call ESMF_TraceRegionEnter("ESMF_RouteHandleCreate() in lieu of ESMF_FieldBundleRegridStore()", rc=rc) + + call ESMF_VMBarrier(VM, rc=rc) + write(msgString,*) "Calling into ESMF_RouteHandleCreate(from RH)..." + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + routehandle(j,i) = ESMF_RouteHandleCreate(routehandle(j,1), & originPetList=originPetList, & targetPetList=targetPetList, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + + call ESMF_VMBarrier(VM, rc=rc) + write(msgString,*) "... returned from ESMF_RouteHandleCreate(from RH)." + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + call ESMF_TraceRegionExit("ESMF_RouteHandleCreate() in lieu of ESMF_FieldBundleRegridStore()", rc=rc) endif - write(msgString,"(A,I2.2,',',I2.2,A)") "... returned from wrtFB(",j,i, ") FieldBundleRegridStore()." + write(msgString,"(A,I2.2,',',I2.2,A)") "... returned from RH creation for wrtFB(",j,i, ")." call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) endif enddo ! j=1, FBcount From fd3e9c90ea4f57ea9b75b53a5586862dcbf88a1e Mon Sep 17 00:00:00 2001 From: Gerhard Theurich Date: Tue, 10 Dec 2024 12:35:41 -0800 Subject: [PATCH 2/3] Log the fcstItemNameList(j) when looping over RH creation... which contains info on regrid method. --- fv3_cap.F90 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fv3_cap.F90 b/fv3_cap.F90 index b01b8b628..0fe2554e3 100644 --- a/fv3_cap.F90 +++ b/fv3_cap.F90 @@ -179,7 +179,7 @@ subroutine InitializeAdvertise(gcomp, rc) type(ESMF_Clock) :: clock character(len=10) :: value - character(240) :: msgString + character(400) :: msgString logical :: isPresent, isSet type(ESMF_VM) :: vm, wrtVM type(ESMF_Time) :: currTime, startTime @@ -786,7 +786,7 @@ subroutine InitializeAdvertise(gcomp, rc) call ESMF_LogSet(flush=.true., rc=rc) - write(msgString,"(A,I2.2,',',I2.2,A)") "RH creation for wrtFB(",j,i, ") ...." + write(msgString,"(A,I2.2,',',I2.2,A)") "RH creation for wrtFB(",j,i, ") ...."//trim(fcstItemNameList(j)) call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) if (i==1) then From ef427b74c817df0a8bbd54219e167bd0aa85b8b9 Mon Sep 17 00:00:00 2001 From: Gerhard Theurich Date: Thu, 12 Dec 2024 08:22:14 -0800 Subject: [PATCH 3/3] Cleanup to be generally usable code. --- fv3_cap.F90 | 37 ++++++++++++------------------------- 1 file changed, 12 insertions(+), 25 deletions(-) diff --git a/fv3_cap.F90 b/fv3_cap.F90 index 0fe2554e3..90ac8811b 100644 --- a/fv3_cap.F90 +++ b/fv3_cap.F90 @@ -179,7 +179,7 @@ subroutine InitializeAdvertise(gcomp, rc) type(ESMF_Clock) :: clock character(len=10) :: value - character(400) :: msgString + character(240) :: msgString logical :: isPresent, isSet type(ESMF_VM) :: vm, wrtVM type(ESMF_Time) :: currTime, startTime @@ -784,8 +784,6 @@ subroutine InitializeAdvertise(gcomp, rc) endif endif - call ESMF_LogSet(flush=.true., rc=rc) - write(msgString,"(A,I2.2,',',I2.2,A)") "RH creation for wrtFB(",j,i, ") ...."//trim(fcstItemNameList(j)) call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) @@ -797,26 +795,25 @@ subroutine InitializeAdvertise(gcomp, rc) if (rh_file_exist .and. use_saved_routehandles) then if(mype==0) print *,'in fv3cap init, routehandle file ',trim(rh_filename), ' exists' - call ESMF_VMBarrier(VM, rc=rc) write(msgString,*) "Calling into ESMF_RouteHandleCreate(from file)...", trim(rh_filename) call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + call ESMF_TraceRegionEnter("ESMF_RouteHandleCreate(from file)", rc=rc) routehandle(j,1) = ESMF_RouteHandleCreate(fileName=trim(rh_filename), rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + call ESMF_TraceRegionExit("ESMF_RouteHandleCreate(from file)", rc=rc) - call ESMF_VMBarrier(VM, rc=rc) write(msgString,*) "... returned from ESMF_RouteHandleCreate(from file)." call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) else ! this is a Store() for the first wrtComp -> must do the Store() if (fieldbundle_uses_redist) then - call ESMF_TraceRegionEnter("ESMF_FieldBundleRedistStore()", rc=rc) - call ESMF_VMBarrier(VM, rc=rc) write(msgString,*) "Calling into FieldBundleRedistStore..." call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + call ESMF_TraceRegionEnter("ESMF_FieldBundleRedistStore()", rc=rc) call ESMF_FieldBundleRedistStore(fcstFB(j), wrtFB(j,1), & routehandle=routehandle(j,1), & rc=rc) @@ -825,21 +822,17 @@ subroutine InitializeAdvertise(gcomp, rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return ! call ESMF_Finalize(endflag=ESMF_END_ABORT) endif + call ESMF_TraceRegionExit("ESMF_FieldBundleRedistStore()", rc=rc) - call ESMF_VMBarrier(VM, rc=rc) write(msgString,*) "... returned from FieldBundleRedistStore." call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) - call ESMF_TraceRegionExit("ESMF_FieldBundleRedistStore()", rc=rc) - call ESMF_LogWrite('af FieldBundleRedistStore', ESMF_LOGMSG_INFO, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return else - call ESMF_TraceRegionEnter("ESMF_FieldBundleRegridStore()", rc=rc) - call ESMF_VMBarrier(VM, rc=rc) write(msgString,*) "Calling into FieldBundleRegridStore..." call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + call ESMF_TraceRegionEnter("ESMF_FieldBundleRegridStore()", rc=rc) call ESMF_FieldBundleRegridStore(fcstFB(j), wrtFB(j,1), & regridMethod=regridmethod, routehandle=routehandle(j,1), & unmappedaction=ESMF_UNMAPPEDACTION_IGNORE, & @@ -848,27 +841,24 @@ subroutine InitializeAdvertise(gcomp, rc) call ESMF_LogWrite('fv3_cap.F90:InitializeAdvertise error in ESMF_FieldBundleRegridStore', ESMF_LOGMSG_ERROR, rc=rc) call ESMF_Finalize(endflag=ESMF_END_ABORT) endif + call ESMF_TraceRegionExit("ESMF_FieldBundleRegridStore()", rc=rc) - call ESMF_VMBarrier(VM, rc=rc) write(msgString,*) "... returned from FieldBundleRegridStore." call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) - call ESMF_TraceRegionExit("ESMF_FieldBundleRegridStore()", rc=rc) - call ESMF_LogWrite('af FieldBundleRegridStore', ESMF_LOGMSG_INFO, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return endif if (use_saved_routehandles) then - call ESMF_VMBarrier(VM, rc=rc) - write(msgString,*) "Calling into ESMF_RouteHandleWrite..." + write(msgString,*) "Calling into ESMF_RouteHandleWrite...", trim(rh_filename) call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + call ESMF_TraceRegionEnter("ESMF_RouteHandleWrite()", rc=rc) call ESMF_RouteHandleWrite(routehandle(j,1), fileName=trim(rh_filename), rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + call ESMF_TraceRegionExit("ESMF_RouteHandleWrite()", rc=rc) if(mype==0) print *,'in fv3cap init, saved routehandle file ',trim(rh_filename) - call ESMF_VMBarrier(VM, rc=rc) write(msgString,*) "... returned from ESMF_RouteHandleWrite." call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) @@ -882,23 +872,20 @@ subroutine InitializeAdvertise(gcomp, rc) else targetPetList(1:num_pes_fcst) = fcstPetList(:) targetPetList(num_pes_fcst+1:) = petList(:) - call ESMF_TraceRegionEnter("ESMF_RouteHandleCreate() in lieu of ESMF_FieldBundleRegridStore()", rc=rc) - call ESMF_VMBarrier(VM, rc=rc) write(msgString,*) "Calling into ESMF_RouteHandleCreate(from RH)..." call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + call ESMF_TraceRegionEnter("ESMF_RouteHandleCreate(from RH) in lieu of ESMF_FieldBundleRegridStore()", rc=rc) routehandle(j,i) = ESMF_RouteHandleCreate(routehandle(j,1), & originPetList=originPetList, & targetPetList=targetPetList, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + call ESMF_TraceRegionExit("ESMF_RouteHandleCreate(from RH) in lieu of ESMF_FieldBundleRegridStore()", rc=rc) - call ESMF_VMBarrier(VM, rc=rc) write(msgString,*) "... returned from ESMF_RouteHandleCreate(from RH)." call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) - call ESMF_TraceRegionExit("ESMF_RouteHandleCreate() in lieu of ESMF_FieldBundleRegridStore()", rc=rc) - endif write(msgString,"(A,I2.2,',',I2.2,A)") "... returned from RH creation for wrtFB(",j,i, ")." call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc)