Merge branch 'mpi_comm_jack' into mpmd_debug

model/src/w3iogomd.F90

@@ -205,7 +205,7 @@ SUBROUTINE S2GRID(S, X, FLDIRN)
     USE W3SERVMD, ONLY : W3S2XY
     USE W3GDATMD, ONLY : NK, UNGTYPE, MAPSF, NTRI, CLGTYPE, RLGTYPE, &
          XGRD, YGRD, SX, SY, X0, Y0, TRIGP, USSP_WN, &
-         NX, NY, NSEA, NBEDGE, EDGES
+        NX, NY, NSEA, NBEDGE, EDGES
 #ifdef W3_SMC
   USE W3SMCOMD, SMCNOVAL=>NOVAL
 #endif
@@ -1386,7 +1386,7 @@ SUBROUTINE W3OUTG ( A, FLPART, FLOUTG, FLOUTG2 )
     USE CONSTANTS
     USE W3GDATMD
     USE W3WDATMD, ONLY: UST, FPIS
-    USE W3ADATMD, ONLY: CG, WN, DW
+    USE W3ADATMD, ONLY: CG, WN, DW, U10, U10D
     USE W3ADATMD, ONLY: HS, WLM, T02, T0M1, T01, FP0,               &
          THM, THS, THP0
     USE W3ADATMD, ONLY: ABA, ABD, UBA, UBD, FCUT, SXX,              &
@@ -1430,13 +1430,20 @@ SUBROUTINE W3OUTG ( A, FLPART, FLOUTG, FLOUTG2 )
     LOGICAL, INTENT(IN)     :: FLPART, FLOUTG, FLOUTG2
     ! MY EDITS 
     INTEGER :: COUNTER
+    ! REAL, ALLOCATABLE       :: X1(:,:)
 #define W3_MPMD
 #ifdef W3_MPMD
   LOGICAL             :: FIRST_STEP = .TRUE., initialized, mpi_initialized_by_us
   integer             :: flag, myproc, nprocs, max_appnum, min_appnum, this_root, other_root, rank_offset, this_nboxes
   integer             :: p, appnum, all_appnum(10), napps, all_argc(10), IERR_MPI
   CHARACTER(LEN=80)   :: exename
   REAL, ALLOCATABLE       :: X1(:,:)
+
+! MY EDITS
+  INTEGER :: n_elements
+  REAL(8), allocatable :: magnitude_values(:)
+  REAL(8), allocatable :: theta_values(:)
+
 #ifdef W3_PDLIB
   REAL(rkind)         :: XY_SEND(NX*NY)
   REAL(rkind)         :: XY_SYNCH_SEND(NSEA)
@@ -1452,6 +1459,10 @@ SUBROUTINE W3OUTG ( A, FLPART, FLOUTG, FLOUTG2 )
     INTEGER                 :: IK, ITH, JSEA, ISEA, IX, IY,         &
          IKP0(NSEAL), NKH(NSEAL),             &
          I, J, LKMS, HKMS, ITL
+
+    ! MY EDITS
+    REAL :: COSU, SINU, TU10, TUDIR
+    INTEGER :: COMMENT
 #ifdef W3_S
     INTEGER, SAVE           :: IENT = 0
 #endif
@@ -2213,7 +2224,12 @@ SUBROUTINE W3OUTG ( A, FLPART, FLOUTG, FLOUTG2 )
      this_root = rank_offset
      other_root = 0
   end if
-
+
+! Uncomment if statement if we only want to receive from ERF
+COMMENT = 0
+
+! if (COMMENT .eq. 1) then
+
   ALLOCATE(X1(NX+1,NY))
 !  ALLOCATE(XY_SEND(NX*NY))
   if (MyProc-1 .eq. this_root) then
@@ -2238,12 +2254,15 @@ SUBROUTINE W3OUTG ( A, FLPART, FLOUTG, FLOUTG2 )
         CALL S2GRID(HS, X1)
         XY_SYNCH_SEND = HS
         CALL SYNCHRONIZE_GLOBAL_ARRAY(XY_SYNCH_SEND)
+
+
         DO JSEA=1, NSEA
            CALL INIT_GET_ISEA(ISEA, JSEA)
            IX     = MAPSF(ISEA,1)
            IY     = MAPSF(ISEA,2)
            XY_SEND((IX)+(IY-1)*NX)=XY_SYNCH_SEND(ISEA)
         END DO
+
         CALL MPI_Send(XY_SEND, NX*NY, MPI_DOUBLE, other_root, 2, MPI_COMM_WORLD, IERR_MPI)
         X1     = UNDEF
         XY_SYNCH_SEND = WLM
@@ -2279,8 +2298,71 @@ SUBROUTINE W3OUTG ( A, FLPART, FLOUTG, FLOUTG2 )
         CALL MPI_Send(XY_SEND, NX*NY, MPI_DOUBLE, other_root, 5, MPI_COMM_WORLD, IERR_MPI)
      end if
   end if
+
+! MY EDITS HERE
+! CHECK XY_SYNCH_SEND, SYNCH_GLOBAL_ARRAY
+    OPEN(5120, file='printmpi.txt', status='unknown', access='append', action="write")
+
+    ! Write HS values to the new file
+    DO JSEA=1, NSEAL
+        CALL INIT_GET_ISEA(ISEA, JSEA)
+        IX     = MAPSF(ISEA,1)
+        IY     = MAPSF(ISEA,2)
+
+        WRITE(5120, *) SIZE(XY_SEND), XY_SEND(ISEA), SIZE(XY_SYNCH_SEND), XY_SYNCH_SEND(ISEA)
+    END DO
+    CLOSE(5120)
   DEALLOCATE(X1)
 !  DEALLOCATE(XY_SEND)
+! end if
+
+! if (COMMENT .eq. 0) then
+
+  if (MyProc-1 .eq. this_root) then
+     if (rank_offset .eq. 0) then !  the first program
+
+        CALL MPI_RECV( n_elements, 1, MPI_INT, other_root, 10, MPI_COMM_WORLD, MPI_STATUS_IGNORE, IERR_MPI );
+        allocate(magnitude_values(n_elements))
+        allocate(theta_values(n_elements))
+
+        CALL MPI_RECV(magnitude_values, n_elements, MPI_DOUBLE, other_root, 12, MPI_COMM_WORLD,MPI_STATUS_IGNORE, IERR_MPI)
+        CALL MPI_RECV(theta_values, n_elements, MPI_DOUBLE, other_root, 14, MPI_COMM_WORLD, MPI_STATUS_IGNORE,IERR_MPI)
+     else ! the second program
+
+        CALL MPI_RECV( n_elements, 1, MPI_INT, other_root, 11, MPI_COMM_WORLD, MPI_STATUS_IGNORE,IERR_MPI );
+        allocate(magnitude_values(n_elements))
+        allocate(theta_values(n_elements))
+
+        call MPI_RECV(magnitude_values, n_elements, MPI_DOUBLE, other_root, 13, MPI_COMM_WORLD,MPI_STATUS_IGNORE, IERR_MPI)
+        call MPI_RECV(theta_values, n_elements, MPI_DOUBLE, other_root, 15, MPI_COMM_WORLD, MPI_STATUS_IGNORE, IERR_MPI)
+     end if
+  end if
+
+
+    print*, "Now I am receiving from ERF"! MPI RECEIVE TEST
+    ! Allocate arrays
+    !allocate(magnitude_values(n_elements))
+    !allocate(theta_values(n_elements))
+    ! Receive magnitude array
+    !call MPI_RECV(magnitude_values, n_elements, MPI_DOUBLE, 0, 3, MPI_COMM_WORLD, IERR_MPI)
+
+    ! Receive theta array
+    !call MPI_RECV(theta_values, n_elements, MPI_DOUBLE, 0, 4, MPI_COMM_WORLD, IERR_MPI)
+    ! Print received values to a txt file
+    open(unit=6123, file='mpi_recv.txt', status='unknown', access='append', action="write")
+     DO JSEA=1, NSEAL
+         CALL INIT_GET_ISEA(ISEA, JSEA)
+         IX     = MAPSF(ISEA,1)
+         IY     = MAPSF(ISEA,2)
+         ! Need correct mapping of magnitude_values and theta_values
+         COUNTER = IX + (IY-1) * NX 
+         WRITE(6123, *) "(", IX, IY, ")", ISEA, COUNTER, size(magnitude_values), magnitude_values(COUNTER), size(theta_values), theta_values(COUNTER), IERR_MPI
+     END DO
+    ! write(6123,*) 'Magnitude Values:', magnitude_values, 'Theta Values:', theta_values
+    close(6123)
+
+! end if
+
 #else
   print*, "Not using MPI this run"
 #endif
@@ -2311,8 +2393,27 @@ SUBROUTINE W3OUTG ( A, FLPART, FLOUTG, FLOUTG2 )
     END DO
     CLOSE(2121)
 
+    ! MY EDITS HERE
+    OPEN(3121, file='output_WND.txt', status='replace', action="write")
+    ! Write (IX,IY) : (x-velocity, y-velocity) values to the new file
+    DO JSEA=1, NSEAL
+        CALL INIT_GET_ISEA(ISEA, JSEA)
+        IX     = MAPSF(ISEA,1)
+        IY     = MAPSF(ISEA,2)
+
+        TU10  = U10(ISEA)                    ! wind velocity U10
+        TUDIR = U10D(ISEA)                   ! wind direction φ (rad)
+        SINU  = SIN(TUDIR)                   ! sinφ
+        COSU  = COS(TUDIR)                   ! cosφ
+
+        ! WRITE(3121, *) "(", IX, IY, ")", "(x_vel, y_vel) = ", "(", COSU, SINU, ") ", "Wind Vel = ", TU10
+
+         WRITE(3121, *) "(", IX, IY, ")", "(x_vel, y_vel) = ", "(", COS(U10D(JSEA)), SIN(U10D(JSEA)), ") ", "Wind Vel = ", U10(JSEA)
+        ! PRINT*, "(", IX, IY, ")", "(x_vel, y_vel) = ", "("COSU, SINU") ", "Wind Vel = ", TU10   
+    END DO
+    CLOSE(3121)
+
 
-    !
     ! 4.  Peak frequencies and directions -------------------------------- *
     ! 4.a Initialize
     !
@@ -4796,6 +4897,8 @@ SUBROUTINE CALC_WBT (A)
       ET    = 0.                           ! ΣE(σ)δσ
       ETP   = 0.                           ! ΣE(σ)δσ at peak only
       !
+
+
       DO IK = 1, NK
         TC     = SIG(IK) / WN(IK, ISEA)  ! phase velocity c=σ/k
         FACTOR = SIG(IK) / CG(IK, ISEA)  ! σ / cg
@@ -4896,6 +4999,27 @@ SUBROUTINE CALC_WBT (A)
       WBT(JSEA) = MIN(1.0, TWBT)
       !
     ENDDO ! JSEA
+
+
+    ! MY EDITS HERE
+    OPEN(3121, file='output_WND.txt', status='replace', action="write")
+    ! Write (IX,IY) : (x-velocity, y-velocity) values to the new file
+    DO JSEA=1, NSEAL
+        CALL INIT_GET_ISEA(ISEA, JSEA)
+        IX     = MAPSF(ISEA,1)
+        IY     = MAPSF(ISEA,2)
+
+        TU10  = U10(ISEA)                    ! wind velocity U10
+        TUDIR = U10D(ISEA)                   ! wind direction φ (rad)
+        SINU  = SIN(TUDIR)                   ! sinφ
+        COSU  = COS(TUDIR)                   ! cosφ
+
+        WRITE(3121, *) "(", IX, IY, ")", "(x_vel, y_vel) = ", "(", COSU, SINU, ") ", "Wind Vel = ", TU10
+        ! PRINT*, "(", IX, IY, ")", "(x_vel, y_vel) = ", "("COSU, SINU") ", "Wind Vel = ", TU10   
+    END DO
+    CLOSE(3121)
+
+
     !/
     !/ End of  CALC_WBT -------------------------------------------------- /
     !/

model/src/w3src4md.F90

@@ -1961,6 +1961,8 @@ SUBROUTINE CALC_USTAR(WINDSPEED,TAUW,USTAR,Z0,CHARN)
         ZZ00 = CHATH*TAUOLD/GRAV
         IF (ZZ0MAX.NE.0) ZZ00=MIN(ZZ00,ZZ0MAX)
         ! Corrects roughness ZZ00 for quasi-linear effect
+        ! MY COMMENTS HERE
+        ! Surface roughness fixed point iteration
         ZZ0 = ZZ00/(1.-X)**XM
         ZNU = 0.11*nu_air/MAX(UST,1E-6)
         ZZ0 = SINTAILPAR(5)*ZNU+ZZ0

model/src/w3srcemd.F90

@@ -682,7 +682,7 @@ SUBROUTINE W3SRCE ( srce_call, IT, ISEA, JSEA, IX, IY, IMOD,          &
     !/ Local parameters
     !/
     INTEGER :: IK, ITH, IS, IS0, NSTEPS, NKH, NKH1, &
-         IKS1, IS1, NSPECH, IDT, IERR, ISP
+         IKS1, IS1, NSPECH, IDT, IERR, ISP, COUNTER
     REAL :: DTTOT, FHIGH, DT, AFILT, DAMAX, AFAC, &
          HDT, ZWND, FP, DEPTH, TAUSCX, TAUSCY, FHIGI
     ! Scaling factor for SIN, SDS, SNL
@@ -1094,7 +1094,15 @@ SUBROUTINE W3SRCE ( srce_call, IT, ISEA, JSEA, IX, IY, IMOD,          &
     TWS = 1./FMEANWS
 #endif
 #ifdef W3_ST6
+
     CALL W3SPR6 (SPEC, CG1, WN1, EMEAN, FMEAN, WNMEAN, AMAX, FP)
+
+! MY EDITS HERE
+    ! open(unit=8123, file='sourceterm.txt', status='unknown', access='append', action="write")
+         print*, " This is the first ifdef for W3_ST6 in w3srce.md "
+    ! WRITE(8123, *) "This is the first ifdef for W3_ST6 in w3srce.md"
+    ! close(8123)
+
 #endif
     !
     ! 1.c2 Stores the initial data
@@ -1842,6 +1850,24 @@ SUBROUTINE W3SRCE ( srce_call, IT, ISEA, JSEA, IX, IY, IMOD,          &
 #endif
       !
 #ifdef W3_ST6
+
+
+    ! MY EDITS HERE
+    COUNTER = 1
+    print*, " Source terms HERE "    
+    IF (COUNTER.EQ.1) THEN
+        OPEN(4121, file='output_SRC.txt', status='replace', action="write")
+        WRITE(4121, *) U10DIR, "(x_vel, y_vel) = ", "(", COS(U10DIR), SIN(U10DIR), ") ", "Wind Vel = ", U10ABS
+
+    ELSE
+
+        WRITE(4121, *) U10DIR, "(x_vel, y_vel) = ", "(", COS(U10DIR), SIN(U10DIR), ") ", "Wind Vel = ", U10ABS
+    ENDIF
+
+    CLOSE(4121)
+    COUNTER = COUNTER + 1
+
+
       IF (FXFM .LE. 0) THEN
         FHIGH = SIG(NK)
       ELSE

model/src/ww3_ounf.F90

@@ -159,7 +159,7 @@ PROGRAM W3OUNF
 
   !/
   USE W3WDATMD, ONLY: W3NDAT, W3SETW
-  USE W3ADATMD, ONLY: W3NAUX, W3SETA
+  USE W3ADATMD, ONLY: W3NAUX, W3SETA, U10, U10D, WBT
   USE W3ODATMD, ONLY: W3NOUT, W3SETO
   USE W3SERVMD, ONLY : ITRACE, NEXTLN, EXTCDE, STR_TO_UPPER
 #ifdef W3_S
@@ -252,6 +252,8 @@ PROGRAM W3OUNF
 #ifdef W3_RTD
   LOGICAL                 :: RTDL = .FALSE.
 #endif
+  INTEGER :: ISEA, JSEA
+  REAL :: TU10, TUDIR
 
   INTEGER                 :: TVARTYPE = NF90_DOUBLE
   CHARACTER(LEN=32)       :: EPOCH_ISO
@@ -680,6 +682,25 @@ PROGRAM W3OUNF
     END IF
     ! END MY EDITS
 
+
+    ! MY EDITS HERE
+    OPEN(3121, file='output_WND.txt', status='replace', action="write")
+    ! Write (IX,IY) : (x-velocity, y-velocity) values to the new file
+    DO JSEA=1, NSEAL
+        ! CALL INIT_GET_ISEA(ISEA, JSEA)
+        !IX     = MAPSF(ISEA,1)
+        !IY     = MAPSF(ISEA,2)
+
+        TU10  = U10(JSEA)                    ! wind velocity U10
+        TUDIR = U10D(JSEA)                   ! wind direction φ (rad)
+       !  SINU  = SIN(TUDIR)                   ! sinφ
+       !  COSU  = COS(TUDIR)                   ! cosφ
+
+        WRITE(3121, *) "(x_vel, y_vel) = ", "(",COS(TUDIR), SIN(TUDIR),") ", "Wind Vel = ", TU10
+      !  PRINT*, "(", IX, IY, ")", "(x_vel, y_vel) = ", "("COSU, SINU") ", "Wind Vel = ", TU10
+    END DO
+    CLOSE(3121)
+
     CALL STME21 ( TOUT , IDTIME )
     WRITE (NDSO,971) IDTIME
 

regtests/ww3_ts2/input/switch_ST1_PR1

@@ -1 +1 @@
-NOGRB SHRD PR1 FLX1 LN1 ST1 NL1 BT0 DB0 TR0 BS0 IC0 IS0 REF0 WNT1 WNX1 CRT1 CRX1 O0 O1 O2 O3 O4 O5 O6 O7 O10 O11
+NOGRB DIST MPI PR1 FLX1 LN1 ST1 NL1 BT0 DB0 TR0 BS0 IC0 IS0 REF0 WNT1 WNX1 CRT1 CRX1 O0 O1 O2 O3 O4 O5 O6 O7 O10 O11

regtests/ww3_ts2/input/switch_ST4_PR1

@@ -1 +1 @@
-NOGRB SHRD PR1 FLX0 LN1 ST4 NL1 BT0 DB0 TR0 BS0 IC0 IS0 REF0 WNT1 WNX1 CRT1 CRX1 O0 O1 O2 O3 O4 O5 O6 O7 O10 O11
+NOGRB DIST MPI PR1 FLX0 LN1 ST4 NL1 BT0 DB0 TR0 BS0 IC0 IS0 REF0 WNT1 WNX1 CRT1 CRX1 O0 O1 O2 O3 O4 O5 O6 O7 O10 O11

regtests/ww3_ts2/input/switch_ST4_PR3_UQ

@@ -1 +1 @@
-NOGRB SHRD PR3 UQ FLX0 LN1 ST4 NL1 BT0 DB0 TR0 BS0 IC0 IS0 REF0 WNT1 WNX1 CRT1 CRX1 O0 O1 O2 O3 O4 O5 O6 O7 O10 O11
+NOGRB DIST MPI PR3 UQ FLX0 LN1 ST4 NL1 BT0 DB0 TR0 BS0 IC0 IS0 REF0 WNT1 WNX1 CRT1 CRX1 O0 O1 O2 O3 O4 O5 O6 O7 O10 O11

regtests/ww3_ts2/input/switch_ST6_PR1

@@ -1 +1 @@
-NOGRB SHRD PR1 FLX4 LN1 ST6 NL1 BT0 DB0 TR0 BS0 IC0 IS0 REF0 WNT1 WNX1 CRT1 CRX1 O0 O1 O2 O3 O4 O5 O6 O7 O10 O11
+NOGRB DIST MPI  PR1 FLX4 LN1 ST6 NL1 BT0 DB0 TR0 BS0 IC0 IS0 REF0 WNT1 WNX1 CRT1 CRX1 O0 O1 O2 O3 O4 O5 O6 O7 O10 O11

regtests/ww3_ts2/input/switch_ST6_PR2_UNO

@@ -1 +1 @@
-NOGRB SHRD PR2 UNO FLX4 LN1 ST6 NL1 BT0 DB0 TR0 BS0 IC0 IS0 REF0 WNT1 WNX1 CRT1 CRX1 O0 O1 O2 O3 O4 O5 O6 O7 O10 O11
+NOGRB DIST MPI PR2 UNO FLX4 LN1 ST6 NL1 BT0 DB0 TR0 BS0 IC0 IS0 REF0 WNT1 WNX1 CRT1 CRX1 O0 O1 O2 O3 O4 O5 O6 O7 O10 O11