From 4abf49044b02be1e7feab9d0b7310974afe6ffd2 Mon Sep 17 00:00:00 2001 From: abishekg7 Date: Mon, 15 Jul 2024 11:55:04 -0600 Subject: [PATCH 01/19] adding some test prints --- src/framework/mpas_stream_manager.F | 1 + 1 file changed, 1 insertion(+) diff --git a/src/framework/mpas_stream_manager.F b/src/framework/mpas_stream_manager.F index d00fcaa800..021bff5543 100644 --- a/src/framework/mpas_stream_manager.F +++ b/src/framework/mpas_stream_manager.F @@ -888,6 +888,7 @@ subroutine MPAS_stream_mgr_add_field(manager, streamID, fieldName, packages, ier return end if + call mpas_log_write('===== my log write stmt .. field is '//trim(fieldName)//' .') ! ! Check that the field does not already exist in the stream ! From 362c44e92adcde737d3f1af246e050cbfd8d9d3d Mon Sep 17 00:00:00 2001 From: abishekg7 Date: Tue, 16 Jul 2024 14:28:10 -0600 Subject: [PATCH 02/19] more prints --- src/framework/mpas_io_streams.F | 63 +++++++++++++++++++++------------ 1 file changed, 41 insertions(+), 22 deletions(-) diff --git a/src/framework/mpas_io_streams.F b/src/framework/mpas_io_streams.F index b445d5881a..ba94b335cc 100644 --- a/src/framework/mpas_io_streams.F +++ b/src/framework/mpas_io_streams.F @@ -3335,14 +3335,14 @@ subroutine MPAS_writeStream(stream, frame, ierr) if (field_cursor % field_type == FIELD_0D_INT) then -!call mpas_log_write('Writing out field '//trim(field_cursor % int0dField % fieldName)) -!call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/field_cursor % isDecomposed/)) -!call mpas_log_write(' > outer dimension size $i', intArgs=(/field_cursor % totalDimSize/)) + call mpas_log_write('Writing out field '//trim(field_cursor % int0dField % fieldName)) + call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/field_cursor % isDecomposed/)) + call mpas_log_write(' > outer dimension size $i', intArgs=(/field_cursor % totalDimSize/)) -!call mpas_log_write('Copying field from first block') + !call mpas_log_write('Copying field from first block') int0d_temp = field_cursor % int0dField % scalar -!call mpas_log_write('MGD calling MPAS_io_put_var now...') + !call mpas_log_write('MGD calling MPAS_io_put_var now...') call MPAS_io_put_var(stream % fileHandle, field_cursor % int0dField % fieldName, int0d_temp, io_err) call MPAS_io_err_mesg(stream % fileHandle % ioContext, io_err, .false.) if (io_err /= MPAS_IO_NOERR .and. present(ierr)) ierr = MPAS_IO_ERR @@ -3357,6 +3357,9 @@ subroutine MPAS_writeStream(stream, frame, ierr) end if do j=1,ncons + call mpas_log_write('Writing out field '//trim(field_cursor % int1dField % fieldName)) + call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/field_cursor % isDecomposed/)) + call mpas_log_write(' > outer dimension size $i', intArgs=(/field_cursor % totalDimSize/)) if (field_cursor % isDecomposed) then ! Gather field from across multiple blocks field_1dint_ptr => field_cursor % int1dField @@ -3376,7 +3379,7 @@ subroutine MPAS_writeStream(stream, frame, ierr) end if if (field_cursor % int1dField % isVarArray) then -! I suspect we will never hit this code, as it doesn't make sense, really + ! I suspect we will never hit this code, as it doesn't make sense, really int0d_temp = field_1dint_ptr % array(j) else int1d_temp(i:i+ownedSize-1) = field_1dint_ptr % array(1:ownedSize) @@ -3424,6 +3427,9 @@ subroutine MPAS_writeStream(stream, frame, ierr) if (field_cursor % isDecomposed) then ! Gather field from across multiple blocks field_2dint_ptr => field_cursor % int2dField + call mpas_log_write('Writing out field '//trim(field_cursor % int2dField % fieldName)) + call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/field_cursor % isDecomposed/)) + call mpas_log_write(' > outer dimension size $i', intArgs=(/field_cursor % totalDimSize/)) i = 1 do while (associated(field_2dint_ptr)) if (trim(field_2dint_ptr % dimNames(2)) == 'nCells') then @@ -3489,6 +3495,9 @@ subroutine MPAS_writeStream(stream, frame, ierr) do j=1,ncons if (field_cursor % isDecomposed) then + call mpas_log_write('Writing out field '//trim(field_cursor % int3dField % fieldName)) + call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/field_cursor % isDecomposed/)) + call mpas_log_write(' > outer dimension size $i', intArgs=(/field_cursor % totalDimSize/)) ! Gather field from across multiple blocks field_3dint_ptr => field_cursor % int3dField i = 1 @@ -3543,14 +3552,14 @@ subroutine MPAS_writeStream(stream, frame, ierr) else if (field_cursor % field_type == FIELD_0D_REAL) then -!call mpas_log_write('Writing out field '//trim(field_cursor % real0dField % fieldName)) -!call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/field_cursor % isDecomposed/)) -!call mpas_log_write(' > outer dimension size $i', intArgs=(/field_cursor % totalDimSize/)) + call mpas_log_write('Writing out field '//trim(field_cursor % real0dField % fieldName)) + call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/field_cursor % isDecomposed/)) + call mpas_log_write(' > outer dimension size $i', intArgs=(/field_cursor % totalDimSize/)) -!call mpas_log_write('Copying field from first block') + !call mpas_log_write('Copying field from first block') real0d_temp = field_cursor % real0dField % scalar -!call mpas_log_write('MGD calling MPAS_io_put_var now...') + !call mpas_log_write('MGD calling MPAS_io_put_var now...') call MPAS_io_put_var(stream % fileHandle, field_cursor % real0dField % fieldName, real0d_temp, io_err) call MPAS_io_err_mesg(stream % fileHandle % ioContext, io_err, .false.) if (io_err /= MPAS_IO_NOERR .and. present(ierr)) ierr = MPAS_IO_ERR @@ -3584,7 +3593,7 @@ subroutine MPAS_writeStream(stream, frame, ierr) end if if (field_cursor % real1dField % isVarArray) then -! I suspect we will never hit this code, as it doesn't make sense, really + ! I suspect we will never hit this code, as it doesn't make sense, really real0d_temp = field_1dreal_ptr % array(j) else real1d_temp(i:i+ownedSize-1) = field_1dreal_ptr % array(1:ownedSize) @@ -3698,6 +3707,9 @@ subroutine MPAS_writeStream(stream, frame, ierr) do j=1,ncons if (field_cursor % isDecomposed) then ! Gather field from across multiple blocks + call mpas_log_write('Writing out field '//trim(field_cursor % real3dField % fieldName)) + call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/field_cursor % isDecomposed/)) + call mpas_log_write(' > outer dimension size $i', intArgs=(/field_cursor % totalDimSize/)) field_3dreal_ptr => field_cursor % real3dField i = 1 do while (associated(field_3dreal_ptr)) @@ -3766,6 +3778,10 @@ subroutine MPAS_writeStream(stream, frame, ierr) do j=1,ncons if (field_cursor % isDecomposed) then + ! Gather field from across multiple blocks + call mpas_log_write('Writing out field '//trim(field_cursor % real4dField % fieldName)) + call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/field_cursor % isDecomposed/)) + call mpas_log_write(' > outer dimension size $i', intArgs=(/field_cursor % totalDimSize/)) ! Gather field from across multiple blocks field_4dreal_ptr => field_cursor % real4dField i = 1 @@ -3837,6 +3853,9 @@ subroutine MPAS_writeStream(stream, frame, ierr) do j=1,ncons if (field_cursor % isDecomposed) then + call mpas_log_write('Writing out field '//trim(field_cursor % real5dField % fieldName)) + call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/field_cursor % isDecomposed/)) + call mpas_log_write(' > outer dimension size $i', intArgs=(/field_cursor % totalDimSize/)) ! Gather field from across multiple blocks field_5dreal_ptr => field_cursor % real5dField i = 1 @@ -3891,24 +3910,24 @@ subroutine MPAS_writeStream(stream, frame, ierr) else if (field_cursor % field_type == FIELD_0D_CHAR) then -!call mpas_log_write('Writing out field '//trim(field_cursor % char0dField % fieldName)) -!call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/field_cursor % isDecomposed/)) -!call mpas_log_write(' > outer dimension size $i', intArgs=(/field_cursor % totalDimSize/)) + call mpas_log_write('Writing out field '//trim(field_cursor % char0dField % fieldName)) + call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/field_cursor % isDecomposed/)) + call mpas_log_write(' > outer dimension size $i', intArgs=(/field_cursor % totalDimSize/)) -!call mpas_log_write('Copying field from first block') -!call mpas_log_write('MGD calling MPAS_io_put_var now...') + !call mpas_log_write('Copying field from first block') + !call mpas_log_write('MGD calling MPAS_io_put_var now...') call MPAS_io_put_var(stream % fileHandle, field_cursor % char0dField % fieldName, field_cursor % char0dField % scalar, io_err) call MPAS_io_err_mesg(stream % fileHandle % ioContext, io_err, .false.) if (io_err /= MPAS_IO_NOERR .and. present(ierr)) ierr = MPAS_IO_ERR else if (field_cursor % field_type == FIELD_1D_CHAR) then -!call mpas_log_write('Writing out field '//trim(field_cursor % char1dField % fieldName)) -!call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/field_cursor % isDecomposed/)) -!call mpas_log_write(' > outer dimension size $i', intArgs=(/field_cursor % totalDimSize/)) + call mpas_log_write('Writing out field '//trim(field_cursor % char1dField % fieldName)) + call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/field_cursor % isDecomposed/)) + call mpas_log_write(' > outer dimension size $i', intArgs=(/field_cursor % totalDimSize/)) -!call mpas_log_write('Copying field from first block') -!call mpas_log_write('MGD calling MPAS_io_put_var now...') + !call mpas_log_write('Copying field from first block') + !call mpas_log_write('MGD calling MPAS_io_put_var now...') call MPAS_io_put_var(stream % fileHandle, field_cursor % char1dField % fieldName, field_cursor % char1dField % array, io_err) call MPAS_io_err_mesg(stream % fileHandle % ioContext, io_err, .false.) if (io_err /= MPAS_IO_NOERR .and. present(ierr)) ierr = MPAS_IO_ERR From 68271e8d5f2b3dbb782277596efc8279e028b686 Mon Sep 17 00:00:00 2001 From: abishekg7 Date: Tue, 16 Jul 2024 15:18:36 -0600 Subject: [PATCH 03/19] more prints --- src/framework/mpas_io_streams.F | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/framework/mpas_io_streams.F b/src/framework/mpas_io_streams.F index ba94b335cc..286bda0f12 100644 --- a/src/framework/mpas_io_streams.F +++ b/src/framework/mpas_io_streams.F @@ -3705,11 +3705,11 @@ subroutine MPAS_writeStream(stream, frame, ierr) end if do j=1,ncons + call mpas_log_write('Writing out real3dField '//trim(field_cursor % real3dField % fieldName)) + call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/field_cursor % isDecomposed/)) + call mpas_log_write(' > outer dimension size $i', intArgs=(/field_cursor % totalDimSize/)) if (field_cursor % isDecomposed) then ! Gather field from across multiple blocks - call mpas_log_write('Writing out field '//trim(field_cursor % real3dField % fieldName)) - call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/field_cursor % isDecomposed/)) - call mpas_log_write(' > outer dimension size $i', intArgs=(/field_cursor % totalDimSize/)) field_3dreal_ptr => field_cursor % real3dField i = 1 do while (associated(field_3dreal_ptr)) @@ -3777,11 +3777,10 @@ subroutine MPAS_writeStream(stream, frame, ierr) end if do j=1,ncons + call mpas_log_write('Writing out real4dField '//trim(field_cursor % real4dField % fieldName)) + call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/field_cursor % isDecomposed/)) + call mpas_log_write(' > outer dimension size $i', intArgs=(/field_cursor % totalDimSize/)) if (field_cursor % isDecomposed) then - ! Gather field from across multiple blocks - call mpas_log_write('Writing out field '//trim(field_cursor % real4dField % fieldName)) - call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/field_cursor % isDecomposed/)) - call mpas_log_write(' > outer dimension size $i', intArgs=(/field_cursor % totalDimSize/)) ! Gather field from across multiple blocks field_4dreal_ptr => field_cursor % real4dField i = 1 @@ -3852,10 +3851,11 @@ subroutine MPAS_writeStream(stream, frame, ierr) end if do j=1,ncons + call mpas_log_write('Writing out real5dField '//trim(field_cursor % real5dField % fieldName)) + call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/field_cursor % isDecomposed/)) + call mpas_log_write(' > outer dimension size $i', intArgs=(/field_cursor % totalDimSize/)) if (field_cursor % isDecomposed) then - call mpas_log_write('Writing out field '//trim(field_cursor % real5dField % fieldName)) - call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/field_cursor % isDecomposed/)) - call mpas_log_write(' > outer dimension size $i', intArgs=(/field_cursor % totalDimSize/)) + ! Gather field from across multiple blocks field_5dreal_ptr => field_cursor % real5dField i = 1 From 7fba5116eb4775c917aeb5509d67d850e122b13b Mon Sep 17 00:00:00 2001 From: abishekg7 Date: Wed, 17 Jul 2024 16:16:04 -0600 Subject: [PATCH 04/19] adding more prints --- src/framework/mpas_stream_manager.F | 87 +++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/src/framework/mpas_stream_manager.F b/src/framework/mpas_stream_manager.F index 021bff5543..04c94f5c97 100644 --- a/src/framework/mpas_stream_manager.F +++ b/src/framework/mpas_stream_manager.F @@ -930,6 +930,73 @@ subroutine MPAS_stream_mgr_add_field(manager, streamID, fieldName, packages, ier end subroutine MPAS_stream_mgr_add_field!}}} + ! subroutine MPAS_stream_mgr_custom(manager, streamID, fieldName, dimNames, dimSizes, packages, ierr)!{{{ + + ! implicit none + + ! character (len=*), parameter :: sub = 'MPAS_stream_mgr_add_field' + + ! type (MPAS_streamManager_type), intent(inout) :: manager + ! character (len=*), intent(in) :: streamID + ! character (len=*), intent(in) :: fieldName + ! character (len=StrKIND), intent(in) dimension(*) :: dimNames + ! integer, intent(in), dimension(*) :: dimSizes + ! character (len=*), intent(in), optional :: packages + ! integer, intent(out), optional :: ierr + + ! type (MPAS_stream_list_type), pointer :: stream + ! type (mpas_pool_field_info_type) :: info + ! character (len=StrKIND) :: field_pkg + ! integer, pointer :: test_ptr + ! logical :: test_logical + ! integer :: err_level + ! integer :: err_local, threadNum + + ! threadNum = mpas_threading_get_thread_num() + + ! STREAM_DEBUG_WRITE('-- Called MPAS_stream_mgr_add_field()') + + ! if (present(ierr)) ierr = MPAS_STREAM_MGR_NOERR + + ! if ( threadNum == 0 ) then + ! ! + ! ! Check that stream exists + ! ! + ! nullify(stream) + ! if (.not. MPAS_stream_list_query(manager % streams, streamID, stream, ierr=err_local)) then + ! STREAM_ERROR_WRITE('Requested stream '//trim(streamID)//' does not exist in stream manager') + ! if (present(ierr)) ierr = MPAS_STREAM_MGR_ERROR + ! return + ! end if + + ! ! + ! ! Don't modify an immutable stream + ! ! + ! if (stream % immutable) then + ! STREAM_ERROR_WRITE('Requested stream '//trim(streamID)//' is immutable.') + ! if (present(ierr)) ierr = MPAS_STREAM_MGR_ERROR + ! return + ! end if + + ! ! + ! ! Check that field exists + ! ! + ! info % nDims = -1 + ! call mpas_pool_get_field_info(manager % allFields, fieldName, info) + ! if (info % nDims == -1) then + ! STREAM_ERROR_WRITE('Requested field '//trim(fieldName)//' not available') + ! if (present(ierr)) ierr = MPAS_STREAM_MGR_ERROR + ! return + ! end if + + ! call mpas_log_write('===== my log write stmt .. field is '//trim(fieldName)//' .') + ! ! + + ! end if + + ! end subroutine MPAS_stream_mgr_add_field!}}} + + !----------------------------------------------------------------------- ! routine MPAS_stream_mgr_add_stream_fields @@ -4332,36 +4399,56 @@ end subroutine gen_random case (0) call mpas_pool_get_field(allFields, itr % memberName, real0d, timeLevel) call MPAS_streamAddField(stream % stream, real0d) + call mpas_log_write('from build_stream: Writing out real0d '//trim(real0d % fieldName)) + call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/real0d % isDecomposed/)) case (1) call mpas_pool_get_field(allFields, itr % memberName, real1d, timeLevel) call MPAS_streamAddField(stream % stream, real1d) + call mpas_log_write('from build_stream: Writing out real1d '//trim(real1d % fieldName)) + call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/real1d % isDecomposed/)) case (2) call mpas_pool_get_field(allFields, itr % memberName, real2d, timeLevel) call MPAS_streamAddField(stream % stream, real2d) + call mpas_log_write('from build_stream: Writing out real2d '//trim(real2d % fieldName)) + call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/real2d % isDecomposed/)) case (3) call mpas_pool_get_field(allFields, itr % memberName, real3d, timeLevel) call MPAS_streamAddField(stream % stream, real3d) + call mpas_log_write('from build_stream: Writing out real3d '//trim(real3d % fieldName)) + call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/real3d % isDecomposed/)) case (4) call mpas_pool_get_field(allFields, itr % memberName, real4d, timeLevel) call MPAS_streamAddField(stream % stream, real4d) + call mpas_log_write('from build_stream: Writing out real4d '//trim(real4d % fieldName)) + call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/real4d % isDecomposed/)) case (5) call mpas_pool_get_field(allFields, itr % memberName, real5d, timeLevel) call MPAS_streamAddField(stream % stream, real5d) + call mpas_log_write('from build_stream: Writing out real5d '//trim(real5d % fieldName)) + call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/real5d % isDecomposed/)) end select case (MPAS_POOL_INTEGER) select case (info % nDims) case (0) call mpas_pool_get_field(allFields, itr % memberName, int0d, timeLevel) call MPAS_streamAddField(stream % stream, int0d) + call mpas_log_write('from build_stream: Writing out int0d '//trim(int0d % fieldName)) + call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/int0d % isDecomposed/)) case (1) call mpas_pool_get_field(allFields, itr % memberName, int1d, timeLevel) call MPAS_streamAddField(stream % stream, int1d) + call mpas_log_write('from build_stream: Writing out int1d '//trim(int1d % fieldName)) + call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/int1d % isDecomposed/)) case (2) call mpas_pool_get_field(allFields, itr % memberName, int2d, timeLevel) call MPAS_streamAddField(stream % stream, int2d) + call mpas_log_write('from build_stream: Writing out int2d '//trim(int2d % fieldName)) + call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/int2d % isDecomposed/)) case (3) call mpas_pool_get_field(allFields, itr % memberName, int3d, timeLevel) call MPAS_streamAddField(stream % stream, int3d) + call mpas_log_write('from build_stream: Writing out int3d '//trim(int3d % fieldName)) + call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/int3d % isDecomposed/)) end select case (MPAS_POOL_CHARACTER) select case (info % nDims) From f94b526e8b93323ea0e7c340b1a3527610318372 Mon Sep 17 00:00:00 2001 From: abishekg7 Date: Thu, 18 Jul 2024 15:20:34 -0600 Subject: [PATCH 05/19] printing dims --- src/framework/mpas_stream_manager.F | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/framework/mpas_stream_manager.F b/src/framework/mpas_stream_manager.F index 04c94f5c97..2d76733863 100644 --- a/src/framework/mpas_stream_manager.F +++ b/src/framework/mpas_stream_manager.F @@ -4400,7 +4400,7 @@ end subroutine gen_random call mpas_pool_get_field(allFields, itr % memberName, real0d, timeLevel) call MPAS_streamAddField(stream % stream, real0d) call mpas_log_write('from build_stream: Writing out real0d '//trim(real0d % fieldName)) - call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/real0d % isDecomposed/)) + call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/real0d % isDecomposed/)) case (1) call mpas_pool_get_field(allFields, itr % memberName, real1d, timeLevel) call MPAS_streamAddField(stream % stream, real1d) @@ -4411,16 +4411,25 @@ end subroutine gen_random call MPAS_streamAddField(stream % stream, real2d) call mpas_log_write('from build_stream: Writing out real2d '//trim(real2d % fieldName)) call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/real2d % isDecomposed/)) + do iDim = 1, 2 + call mpas_log_write('from build_stream: Dimname '//trim(real2d % dimNames(iDim))//' Size '//trim(real2d % dimSizes(iDim))) + end do case (3) call mpas_pool_get_field(allFields, itr % memberName, real3d, timeLevel) call MPAS_streamAddField(stream % stream, real3d) call mpas_log_write('from build_stream: Writing out real3d '//trim(real3d % fieldName)) call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/real3d % isDecomposed/)) + do iDim = 1, 3 + call mpas_log_write('from build_stream: Dimname '//trim(real2d % dimNames(iDim))//' Size '//trim(real2d % dimSizes(iDim))) + end do case (4) call mpas_pool_get_field(allFields, itr % memberName, real4d, timeLevel) call MPAS_streamAddField(stream % stream, real4d) call mpas_log_write('from build_stream: Writing out real4d '//trim(real4d % fieldName)) call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/real4d % isDecomposed/)) + do iDim = 1, 4 + call mpas_log_write('from build_stream: Dimname '//trim(real2d % dimNames(iDim))//' Size '//trim(real2d % dimSizes(iDim))) + end do case (5) call mpas_pool_get_field(allFields, itr % memberName, real5d, timeLevel) call MPAS_streamAddField(stream % stream, real5d) From d438c29648125908e92fd3449e96bf0d1c2b11b5 Mon Sep 17 00:00:00 2001 From: abishekg7 Date: Thu, 18 Jul 2024 15:25:13 -0600 Subject: [PATCH 06/19] printing dims --- src/framework/mpas_stream_manager.F | 1 + 1 file changed, 1 insertion(+) diff --git a/src/framework/mpas_stream_manager.F b/src/framework/mpas_stream_manager.F index 2d76733863..0e66204e2b 100644 --- a/src/framework/mpas_stream_manager.F +++ b/src/framework/mpas_stream_manager.F @@ -4308,6 +4308,7 @@ end subroutine gen_random character (len=StrKIND), pointer :: packages logical :: active_field integer :: err_level + integer :: iDim if (direction == MPAS_STREAM_OUTPUT) then From 6ac4a3ba92f0e38bfe384ee6138575edaef147c0 Mon Sep 17 00:00:00 2001 From: abishekg7 Date: Thu, 18 Jul 2024 15:30:34 -0600 Subject: [PATCH 07/19] printing dims --- src/framework/mpas_stream_manager.F | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/framework/mpas_stream_manager.F b/src/framework/mpas_stream_manager.F index 0e66204e2b..01b9d29182 100644 --- a/src/framework/mpas_stream_manager.F +++ b/src/framework/mpas_stream_manager.F @@ -4421,7 +4421,7 @@ end subroutine gen_random call mpas_log_write('from build_stream: Writing out real3d '//trim(real3d % fieldName)) call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/real3d % isDecomposed/)) do iDim = 1, 3 - call mpas_log_write('from build_stream: Dimname '//trim(real2d % dimNames(iDim))//' Size '//trim(real2d % dimSizes(iDim))) + call mpas_log_write('from build_stream: Dimname '//trim(real3d % dimNames(iDim))//' Size '//trim(real3d % dimSizes(iDim))) end do case (4) call mpas_pool_get_field(allFields, itr % memberName, real4d, timeLevel) @@ -4429,7 +4429,7 @@ end subroutine gen_random call mpas_log_write('from build_stream: Writing out real4d '//trim(real4d % fieldName)) call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/real4d % isDecomposed/)) do iDim = 1, 4 - call mpas_log_write('from build_stream: Dimname '//trim(real2d % dimNames(iDim))//' Size '//trim(real2d % dimSizes(iDim))) + call mpas_log_write('from build_stream: Dimname '//trim(real4d % dimNames(iDim))//' Size '//trim(real4d % dimSizes(iDim))) end do case (5) call mpas_pool_get_field(allFields, itr % memberName, real5d, timeLevel) From 7b4aa9b6a51082011e066a2c3a49fc2f1f7e4bc5 Mon Sep 17 00:00:00 2001 From: abishekg7 Date: Thu, 18 Jul 2024 15:35:08 -0600 Subject: [PATCH 08/19] printing dims --- src/framework/mpas_stream_manager.F | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/framework/mpas_stream_manager.F b/src/framework/mpas_stream_manager.F index 01b9d29182..c3ec169ac3 100644 --- a/src/framework/mpas_stream_manager.F +++ b/src/framework/mpas_stream_manager.F @@ -4413,7 +4413,7 @@ end subroutine gen_random call mpas_log_write('from build_stream: Writing out real2d '//trim(real2d % fieldName)) call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/real2d % isDecomposed/)) do iDim = 1, 2 - call mpas_log_write('from build_stream: Dimname '//trim(real2d % dimNames(iDim))//' Size '//trim(real2d % dimSizes(iDim))) + call mpas_log_write('from build_stream: Dimname '//trim(real2d % dimNames(iDim))//' Size $i',intArgs=(/real2d % dimSizes(iDim)/))) end do case (3) call mpas_pool_get_field(allFields, itr % memberName, real3d, timeLevel) @@ -4421,7 +4421,7 @@ end subroutine gen_random call mpas_log_write('from build_stream: Writing out real3d '//trim(real3d % fieldName)) call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/real3d % isDecomposed/)) do iDim = 1, 3 - call mpas_log_write('from build_stream: Dimname '//trim(real3d % dimNames(iDim))//' Size '//trim(real3d % dimSizes(iDim))) + call mpas_log_write('from build_stream: Dimname '//trim(real3d % dimNames(iDim))//' Size $i',intArgs=(/real3d % dimSizes(iDim)/))) end do case (4) call mpas_pool_get_field(allFields, itr % memberName, real4d, timeLevel) @@ -4429,7 +4429,7 @@ end subroutine gen_random call mpas_log_write('from build_stream: Writing out real4d '//trim(real4d % fieldName)) call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/real4d % isDecomposed/)) do iDim = 1, 4 - call mpas_log_write('from build_stream: Dimname '//trim(real4d % dimNames(iDim))//' Size '//trim(real4d % dimSizes(iDim))) + call mpas_log_write('from build_stream: Dimname '//trim(real4d % dimNames(iDim))//' Size $i',intArgs=(/real4d % dimSizes(iDim)/))) end do case (5) call mpas_pool_get_field(allFields, itr % memberName, real5d, timeLevel) From 2107e919dd4dcc9de3be570817ba24ec8d9898eb Mon Sep 17 00:00:00 2001 From: abishekg7 Date: Thu, 18 Jul 2024 15:39:48 -0600 Subject: [PATCH 09/19] printing dims --- src/framework/mpas_stream_manager.F | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/framework/mpas_stream_manager.F b/src/framework/mpas_stream_manager.F index c3ec169ac3..ad9cff671a 100644 --- a/src/framework/mpas_stream_manager.F +++ b/src/framework/mpas_stream_manager.F @@ -4413,7 +4413,7 @@ end subroutine gen_random call mpas_log_write('from build_stream: Writing out real2d '//trim(real2d % fieldName)) call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/real2d % isDecomposed/)) do iDim = 1, 2 - call mpas_log_write('from build_stream: Dimname '//trim(real2d % dimNames(iDim))//' Size $i',intArgs=(/real2d % dimSizes(iDim)/))) + call mpas_log_write('from build_stream: Dimname '//trim(real2d % dimNames(iDim))//' Size $i',intArgs=(/real2d % dimSizes(iDim)/)) end do case (3) call mpas_pool_get_field(allFields, itr % memberName, real3d, timeLevel) @@ -4421,7 +4421,7 @@ end subroutine gen_random call mpas_log_write('from build_stream: Writing out real3d '//trim(real3d % fieldName)) call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/real3d % isDecomposed/)) do iDim = 1, 3 - call mpas_log_write('from build_stream: Dimname '//trim(real3d % dimNames(iDim))//' Size $i',intArgs=(/real3d % dimSizes(iDim)/))) + call mpas_log_write('from build_stream: Dimname '//trim(real3d % dimNames(iDim))//' Size $i',intArgs=(/real3d % dimSizes(iDim)/)) end do case (4) call mpas_pool_get_field(allFields, itr % memberName, real4d, timeLevel) @@ -4429,7 +4429,7 @@ end subroutine gen_random call mpas_log_write('from build_stream: Writing out real4d '//trim(real4d % fieldName)) call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/real4d % isDecomposed/)) do iDim = 1, 4 - call mpas_log_write('from build_stream: Dimname '//trim(real4d % dimNames(iDim))//' Size $i',intArgs=(/real4d % dimSizes(iDim)/))) + call mpas_log_write('from build_stream: Dimname '//trim(real4d % dimNames(iDim))//' Size $i',intArgs=(/real4d % dimSizes(iDim)/)) end do case (5) call mpas_pool_get_field(allFields, itr % memberName, real5d, timeLevel) From d5972cd9715eb50b13da18578838e2184381d6c1 Mon Sep 17 00:00:00 2001 From: abishekg7 Date: Fri, 19 Jul 2024 14:47:52 -0600 Subject: [PATCH 10/19] testing code --- src/framework/mpas_stream_manager.F | 103 ++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) diff --git a/src/framework/mpas_stream_manager.F b/src/framework/mpas_stream_manager.F index ad9cff671a..fa8125486a 100644 --- a/src/framework/mpas_stream_manager.F +++ b/src/framework/mpas_stream_manager.F @@ -951,6 +951,9 @@ end subroutine MPAS_stream_mgr_add_field!}}} ! logical :: test_logical ! integer :: err_level ! integer :: err_local, threadNum + ! integer :: size_field, loc_dim_size, global_dim_size + + ! size_field = 1 ! threadNum = mpas_threading_get_thread_num() @@ -958,6 +961,93 @@ end subroutine MPAS_stream_mgr_add_field!}}} ! if (present(ierr)) ierr = MPAS_STREAM_MGR_NOERR + ! select case (info % fieldType) + ! case (MPAS_POOL_REAL) + ! select case (info % nDims) + ! case (0) + ! call mpas_pool_get_field(allFields, itr % memberName, real0d, timeLevel) + ! call MPAS_streamAddField(stream % stream, real0d) + ! call mpas_log_write('from build_stream: Writing out real0d '//trim(real0d % fieldName)) + ! call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/real0d % isDecomposed/)) + + + ! case (1) + ! call mpas_pool_get_field(allFields, itr % memberName, real1d, timeLevel) + ! call MPAS_streamAddField(stream % stream, real1d) + ! call mpas_log_write('from build_stream: Writing out real1d '//trim(real1d % fieldName)) + ! call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/real1d % isDecomposed/)) + ! real1d % block % domain % dminfo + ! case (2) + ! call mpas_pool_get_field(allFields, itr % memberName, real2d, timeLevel) + ! call MPAS_streamAddField(stream % stream, real2d) + ! call mpas_log_write('from build_stream: Writing out real2d '//trim(real2d % fieldName)) + ! call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/real2d % isDecomposed/)) + ! do iDim = 1, 2 + ! call mpas_log_write('from build_stream: Dimname '//trim(real2d % dimNames(iDim))//' Size $i',intArgs=(/real2d % dimSizes(iDim)/)) + ! loc_dim_size = real2d % dimSizes(iDim) + ! if ( is_decomposed_dim(real2d % dimNames(iDim))) then + ! call mpas_dmpar_sum_int8(real2d % block % domain % dminfo, loc_dim_size, global_dim_size) + ! size_field = size_field * global_dim_size + ! else + ! size_field = size_field * loc_dim_size + ! end if + ! end do + ! case (3) + ! call mpas_pool_get_field(allFields, itr % memberName, real3d, timeLevel) + ! call MPAS_streamAddField(stream % stream, real3d) + ! call mpas_log_write('from build_stream: Writing out real3d '//trim(real3d % fieldName)) + ! call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/real3d % isDecomposed/)) + ! do iDim = 1, 3 + ! call mpas_log_write('from build_stream: Dimname '//trim(real3d % dimNames(iDim))//' Size $i',intArgs=(/real3d % dimSizes(iDim)/)) + ! end do + ! case (4) + ! call mpas_pool_get_field(allFields, itr % memberName, real4d, timeLevel) + ! call MPAS_streamAddField(stream % stream, real4d) + ! call mpas_log_write('from build_stream: Writing out real4d '//trim(real4d % fieldName)) + ! call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/real4d % isDecomposed/)) + ! do iDim = 1, 4 + ! call mpas_log_write('from build_stream: Dimname '//trim(real4d % dimNames(iDim))//' Size $i',intArgs=(/real4d % dimSizes(iDim)/)) + ! end do + ! case (5) + ! call mpas_pool_get_field(allFields, itr % memberName, real5d, timeLevel) + ! call MPAS_streamAddField(stream % stream, real5d) + ! call mpas_log_write('from build_stream: Writing out real5d '//trim(real5d % fieldName)) + ! call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/real5d % isDecomposed/)) + ! end select + ! case (MPAS_POOL_INTEGER) + ! select case (info % nDims) + ! case (0) + ! call mpas_pool_get_field(allFields, itr % memberName, int0d, timeLevel) + ! call MPAS_streamAddField(stream % stream, int0d) + ! call mpas_log_write('from build_stream: Writing out int0d '//trim(int0d % fieldName)) + ! call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/int0d % isDecomposed/)) + ! case (1) + ! call mpas_pool_get_field(allFields, itr % memberName, int1d, timeLevel) + ! call MPAS_streamAddField(stream % stream, int1d) + ! call mpas_log_write('from build_stream: Writing out int1d '//trim(int1d % fieldName)) + ! call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/int1d % isDecomposed/)) + ! case (2) + ! call mpas_pool_get_field(allFields, itr % memberName, int2d, timeLevel) + ! call MPAS_streamAddField(stream % stream, int2d) + ! call mpas_log_write('from build_stream: Writing out int2d '//trim(int2d % fieldName)) + ! call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/int2d % isDecomposed/)) + ! case (3) + ! call mpas_pool_get_field(allFields, itr % memberName, int3d, timeLevel) + ! call MPAS_streamAddField(stream % stream, int3d) + ! call mpas_log_write('from build_stream: Writing out int3d '//trim(int3d % fieldName)) + ! call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/int3d % isDecomposed/)) + ! end select + ! case (MPAS_POOL_CHARACTER) + ! select case (info % nDims) + ! case (0) + ! call mpas_pool_get_field(allFields, itr % memberName, char0d, timeLevel) + ! call MPAS_streamAddField(stream % stream, char0d) + ! case (1) + ! call mpas_pool_get_field(allFields, itr % memberName, char1d, timeLevel) + ! call MPAS_streamAddField(stream % stream, char1d) + ! end select + ! end select + ! if ( threadNum == 0 ) then ! ! ! ! Check that stream exists @@ -4309,6 +4399,7 @@ end subroutine gen_random logical :: active_field integer :: err_level integer :: iDim + integer :: size_field, loc_dim_size, global_dim_size if (direction == MPAS_STREAM_OUTPUT) then @@ -4394,6 +4485,8 @@ end subroutine gen_random timeLevel = 1 end if + size_field = 1 + select case (info % fieldType) case (MPAS_POOL_REAL) select case (info % nDims) @@ -4413,8 +4506,18 @@ end subroutine gen_random call mpas_log_write('from build_stream: Writing out real2d '//trim(real2d % fieldName)) call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/real2d % isDecomposed/)) do iDim = 1, 2 + loc_dim_size = real2d % dimSizes(iDim) call mpas_log_write('from build_stream: Dimname '//trim(real2d % dimNames(iDim))//' Size $i',intArgs=(/real2d % dimSizes(iDim)/)) + if ( is_decomposed_dim(real2d % dimNames(iDim))) then + call mpas_dmpar_sum_int8(real2d % block % domain % dminfo, loc_dim_size, global_dim_size) + size_field = size_field * global_dim_size + call mpas_log_write('from build_stream ----- Dimname is decomposed '//trim(real2d % dimNames(iDim))//' local size $i global size $i ',intArgs=(/loc_dim_size, global_dim_size/)) + else + size_field = size_field * loc_dim_size + call mpas_log_write('from build_stream ----- Dimname is not decomposed '//trim(real2d % dimNames(iDim))//' Size $i',intArgs=(/loc_dim_size/)) + end if end do + call mpas_log_write('from build_stream ----- cum size is $i',intArgs=(/size_field/)) case (3) call mpas_pool_get_field(allFields, itr % memberName, real3d, timeLevel) call MPAS_streamAddField(stream % stream, real3d) From 1a784a511ee050bdfda4fe580a1e97cbec409793 Mon Sep 17 00:00:00 2001 From: Abishek Gopal Date: Fri, 19 Jul 2024 18:33:13 -0600 Subject: [PATCH 11/19] Adding function global_dim_size --- src/framework/mpas_stream_manager.F | 40 ++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/src/framework/mpas_stream_manager.F b/src/framework/mpas_stream_manager.F index fa8125486a..667292404e 100644 --- a/src/framework/mpas_stream_manager.F +++ b/src/framework/mpas_stream_manager.F @@ -4399,7 +4399,8 @@ end subroutine gen_random logical :: active_field integer :: err_level integer :: iDim - integer :: size_field, loc_dim_size, global_dim_size + !integer(kind=I8KIND) :: size_field, loc_dim_size, global_dim_size + integer :: size_field, loc_dim_size if (direction == MPAS_STREAM_OUTPUT) then @@ -4453,6 +4454,8 @@ end subroutine gen_random call mpas_pool_begin_iteration(stream % field_pool) + + FIELD_LOOP: do while ( mpas_pool_get_next_member(stream % field_pool, itr) ) if (itr % memberType == MPAS_POOL_CONFIG) then @@ -4505,13 +4508,15 @@ end subroutine gen_random call MPAS_streamAddField(stream % stream, real2d) call mpas_log_write('from build_stream: Writing out real2d '//trim(real2d % fieldName)) call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/real2d % isDecomposed/)) + do iDim = 1, 2 loc_dim_size = real2d % dimSizes(iDim) call mpas_log_write('from build_stream: Dimname '//trim(real2d % dimNames(iDim))//' Size $i',intArgs=(/real2d % dimSizes(iDim)/)) if ( is_decomposed_dim(real2d % dimNames(iDim))) then - call mpas_dmpar_sum_int8(real2d % block % domain % dminfo, loc_dim_size, global_dim_size) - size_field = size_field * global_dim_size - call mpas_log_write('from build_stream ----- Dimname is decomposed '//trim(real2d % dimNames(iDim))//' local size $i global size $i ',intArgs=(/loc_dim_size, global_dim_size/)) + !call mpas_dmpar_sum_int(real2d % block % domain % dminfo, loc_dim_size, global_dim_size) + size_field = size_field * global_dim_size(real2d % block, real2d % dimNames(iDim)) + !size_field = size_field * global_dim_size + else size_field = size_field * loc_dim_size call mpas_log_write('from build_stream ----- Dimname is not decomposed '//trim(real2d % dimNames(iDim))//' Size $i',intArgs=(/loc_dim_size/)) @@ -5046,6 +5051,33 @@ logical function is_decomposed_dim(dimName) !{{{ end function is_decomposed_dim !}}} + + + integer function global_dim_size(block, dimName) !{{{ + + implicit none + + character(len=*), intent(in) :: dimName + type(block_type), intent(in) :: block + integer, pointer :: loc_dim_size + + if (trim(dimName) == 'nCells') then + call mpas_pool_get_dimension(block % dimensions, 'nCellsSolve', loc_dim_size) + else if (trim(dimName) == 'nEdges') then + call mpas_pool_get_dimension(block % dimensions, 'nEdgesSolve', loc_dim_size) + else if (trim(dimName) == 'nVertices') then + call mpas_pool_get_dimension(block % dimensions, 'nVerticesSolve', loc_dim_size) + else + global_dim_size = -1 + end if + + call mpas_dmpar_sum_int(block % domain % dminfo, loc_dim_size, global_dim_size) + call mpas_log_write('from build_stream ----- Dimname is decomposed '//trim(dimName)//' local size $i global size $i ',intArgs=(/loc_dim_size, global_dim_size/)) + +end function global_dim_size !}}} + + + !----------------------------------------------------------------------- ! routine prewrite_reindex From ea8d1c81f2f1831bdc0cd0e8c30958c08467c854 Mon Sep 17 00:00:00 2001 From: Abishek Gopal Date: Mon, 22 Jul 2024 11:26:15 -0600 Subject: [PATCH 12/19] some cleanup --- src/framework/mpas_io_streams.F | 27 ---- src/framework/mpas_stream_manager.F | 193 +++++----------------------- 2 files changed, 30 insertions(+), 190 deletions(-) diff --git a/src/framework/mpas_io_streams.F b/src/framework/mpas_io_streams.F index 286bda0f12..f48b2d1c48 100644 --- a/src/framework/mpas_io_streams.F +++ b/src/framework/mpas_io_streams.F @@ -3357,9 +3357,6 @@ subroutine MPAS_writeStream(stream, frame, ierr) end if do j=1,ncons - call mpas_log_write('Writing out field '//trim(field_cursor % int1dField % fieldName)) - call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/field_cursor % isDecomposed/)) - call mpas_log_write(' > outer dimension size $i', intArgs=(/field_cursor % totalDimSize/)) if (field_cursor % isDecomposed) then ! Gather field from across multiple blocks field_1dint_ptr => field_cursor % int1dField @@ -3427,9 +3424,6 @@ subroutine MPAS_writeStream(stream, frame, ierr) if (field_cursor % isDecomposed) then ! Gather field from across multiple blocks field_2dint_ptr => field_cursor % int2dField - call mpas_log_write('Writing out field '//trim(field_cursor % int2dField % fieldName)) - call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/field_cursor % isDecomposed/)) - call mpas_log_write(' > outer dimension size $i', intArgs=(/field_cursor % totalDimSize/)) i = 1 do while (associated(field_2dint_ptr)) if (trim(field_2dint_ptr % dimNames(2)) == 'nCells') then @@ -3495,9 +3489,6 @@ subroutine MPAS_writeStream(stream, frame, ierr) do j=1,ncons if (field_cursor % isDecomposed) then - call mpas_log_write('Writing out field '//trim(field_cursor % int3dField % fieldName)) - call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/field_cursor % isDecomposed/)) - call mpas_log_write(' > outer dimension size $i', intArgs=(/field_cursor % totalDimSize/)) ! Gather field from across multiple blocks field_3dint_ptr => field_cursor % int3dField i = 1 @@ -3552,9 +3543,6 @@ subroutine MPAS_writeStream(stream, frame, ierr) else if (field_cursor % field_type == FIELD_0D_REAL) then - call mpas_log_write('Writing out field '//trim(field_cursor % real0dField % fieldName)) - call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/field_cursor % isDecomposed/)) - call mpas_log_write(' > outer dimension size $i', intArgs=(/field_cursor % totalDimSize/)) !call mpas_log_write('Copying field from first block') real0d_temp = field_cursor % real0dField % scalar @@ -3705,9 +3693,6 @@ subroutine MPAS_writeStream(stream, frame, ierr) end if do j=1,ncons - call mpas_log_write('Writing out real3dField '//trim(field_cursor % real3dField % fieldName)) - call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/field_cursor % isDecomposed/)) - call mpas_log_write(' > outer dimension size $i', intArgs=(/field_cursor % totalDimSize/)) if (field_cursor % isDecomposed) then ! Gather field from across multiple blocks field_3dreal_ptr => field_cursor % real3dField @@ -3777,9 +3762,6 @@ subroutine MPAS_writeStream(stream, frame, ierr) end if do j=1,ncons - call mpas_log_write('Writing out real4dField '//trim(field_cursor % real4dField % fieldName)) - call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/field_cursor % isDecomposed/)) - call mpas_log_write(' > outer dimension size $i', intArgs=(/field_cursor % totalDimSize/)) if (field_cursor % isDecomposed) then ! Gather field from across multiple blocks field_4dreal_ptr => field_cursor % real4dField @@ -3851,9 +3833,6 @@ subroutine MPAS_writeStream(stream, frame, ierr) end if do j=1,ncons - call mpas_log_write('Writing out real5dField '//trim(field_cursor % real5dField % fieldName)) - call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/field_cursor % isDecomposed/)) - call mpas_log_write(' > outer dimension size $i', intArgs=(/field_cursor % totalDimSize/)) if (field_cursor % isDecomposed) then ! Gather field from across multiple blocks @@ -3910,9 +3889,6 @@ subroutine MPAS_writeStream(stream, frame, ierr) else if (field_cursor % field_type == FIELD_0D_CHAR) then - call mpas_log_write('Writing out field '//trim(field_cursor % char0dField % fieldName)) - call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/field_cursor % isDecomposed/)) - call mpas_log_write(' > outer dimension size $i', intArgs=(/field_cursor % totalDimSize/)) !call mpas_log_write('Copying field from first block') !call mpas_log_write('MGD calling MPAS_io_put_var now...') @@ -3922,9 +3898,6 @@ subroutine MPAS_writeStream(stream, frame, ierr) else if (field_cursor % field_type == FIELD_1D_CHAR) then - call mpas_log_write('Writing out field '//trim(field_cursor % char1dField % fieldName)) - call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/field_cursor % isDecomposed/)) - call mpas_log_write(' > outer dimension size $i', intArgs=(/field_cursor % totalDimSize/)) !call mpas_log_write('Copying field from first block') !call mpas_log_write('MGD calling MPAS_io_put_var now...') diff --git a/src/framework/mpas_stream_manager.F b/src/framework/mpas_stream_manager.F index 667292404e..630eee1a71 100644 --- a/src/framework/mpas_stream_manager.F +++ b/src/framework/mpas_stream_manager.F @@ -930,163 +930,6 @@ subroutine MPAS_stream_mgr_add_field(manager, streamID, fieldName, packages, ier end subroutine MPAS_stream_mgr_add_field!}}} - ! subroutine MPAS_stream_mgr_custom(manager, streamID, fieldName, dimNames, dimSizes, packages, ierr)!{{{ - - ! implicit none - - ! character (len=*), parameter :: sub = 'MPAS_stream_mgr_add_field' - - ! type (MPAS_streamManager_type), intent(inout) :: manager - ! character (len=*), intent(in) :: streamID - ! character (len=*), intent(in) :: fieldName - ! character (len=StrKIND), intent(in) dimension(*) :: dimNames - ! integer, intent(in), dimension(*) :: dimSizes - ! character (len=*), intent(in), optional :: packages - ! integer, intent(out), optional :: ierr - - ! type (MPAS_stream_list_type), pointer :: stream - ! type (mpas_pool_field_info_type) :: info - ! character (len=StrKIND) :: field_pkg - ! integer, pointer :: test_ptr - ! logical :: test_logical - ! integer :: err_level - ! integer :: err_local, threadNum - ! integer :: size_field, loc_dim_size, global_dim_size - - ! size_field = 1 - - ! threadNum = mpas_threading_get_thread_num() - - ! STREAM_DEBUG_WRITE('-- Called MPAS_stream_mgr_add_field()') - - ! if (present(ierr)) ierr = MPAS_STREAM_MGR_NOERR - - ! select case (info % fieldType) - ! case (MPAS_POOL_REAL) - ! select case (info % nDims) - ! case (0) - ! call mpas_pool_get_field(allFields, itr % memberName, real0d, timeLevel) - ! call MPAS_streamAddField(stream % stream, real0d) - ! call mpas_log_write('from build_stream: Writing out real0d '//trim(real0d % fieldName)) - ! call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/real0d % isDecomposed/)) - - - ! case (1) - ! call mpas_pool_get_field(allFields, itr % memberName, real1d, timeLevel) - ! call MPAS_streamAddField(stream % stream, real1d) - ! call mpas_log_write('from build_stream: Writing out real1d '//trim(real1d % fieldName)) - ! call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/real1d % isDecomposed/)) - ! real1d % block % domain % dminfo - ! case (2) - ! call mpas_pool_get_field(allFields, itr % memberName, real2d, timeLevel) - ! call MPAS_streamAddField(stream % stream, real2d) - ! call mpas_log_write('from build_stream: Writing out real2d '//trim(real2d % fieldName)) - ! call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/real2d % isDecomposed/)) - ! do iDim = 1, 2 - ! call mpas_log_write('from build_stream: Dimname '//trim(real2d % dimNames(iDim))//' Size $i',intArgs=(/real2d % dimSizes(iDim)/)) - ! loc_dim_size = real2d % dimSizes(iDim) - ! if ( is_decomposed_dim(real2d % dimNames(iDim))) then - ! call mpas_dmpar_sum_int8(real2d % block % domain % dminfo, loc_dim_size, global_dim_size) - ! size_field = size_field * global_dim_size - ! else - ! size_field = size_field * loc_dim_size - ! end if - ! end do - ! case (3) - ! call mpas_pool_get_field(allFields, itr % memberName, real3d, timeLevel) - ! call MPAS_streamAddField(stream % stream, real3d) - ! call mpas_log_write('from build_stream: Writing out real3d '//trim(real3d % fieldName)) - ! call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/real3d % isDecomposed/)) - ! do iDim = 1, 3 - ! call mpas_log_write('from build_stream: Dimname '//trim(real3d % dimNames(iDim))//' Size $i',intArgs=(/real3d % dimSizes(iDim)/)) - ! end do - ! case (4) - ! call mpas_pool_get_field(allFields, itr % memberName, real4d, timeLevel) - ! call MPAS_streamAddField(stream % stream, real4d) - ! call mpas_log_write('from build_stream: Writing out real4d '//trim(real4d % fieldName)) - ! call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/real4d % isDecomposed/)) - ! do iDim = 1, 4 - ! call mpas_log_write('from build_stream: Dimname '//trim(real4d % dimNames(iDim))//' Size $i',intArgs=(/real4d % dimSizes(iDim)/)) - ! end do - ! case (5) - ! call mpas_pool_get_field(allFields, itr % memberName, real5d, timeLevel) - ! call MPAS_streamAddField(stream % stream, real5d) - ! call mpas_log_write('from build_stream: Writing out real5d '//trim(real5d % fieldName)) - ! call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/real5d % isDecomposed/)) - ! end select - ! case (MPAS_POOL_INTEGER) - ! select case (info % nDims) - ! case (0) - ! call mpas_pool_get_field(allFields, itr % memberName, int0d, timeLevel) - ! call MPAS_streamAddField(stream % stream, int0d) - ! call mpas_log_write('from build_stream: Writing out int0d '//trim(int0d % fieldName)) - ! call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/int0d % isDecomposed/)) - ! case (1) - ! call mpas_pool_get_field(allFields, itr % memberName, int1d, timeLevel) - ! call MPAS_streamAddField(stream % stream, int1d) - ! call mpas_log_write('from build_stream: Writing out int1d '//trim(int1d % fieldName)) - ! call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/int1d % isDecomposed/)) - ! case (2) - ! call mpas_pool_get_field(allFields, itr % memberName, int2d, timeLevel) - ! call MPAS_streamAddField(stream % stream, int2d) - ! call mpas_log_write('from build_stream: Writing out int2d '//trim(int2d % fieldName)) - ! call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/int2d % isDecomposed/)) - ! case (3) - ! call mpas_pool_get_field(allFields, itr % memberName, int3d, timeLevel) - ! call MPAS_streamAddField(stream % stream, int3d) - ! call mpas_log_write('from build_stream: Writing out int3d '//trim(int3d % fieldName)) - ! call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/int3d % isDecomposed/)) - ! end select - ! case (MPAS_POOL_CHARACTER) - ! select case (info % nDims) - ! case (0) - ! call mpas_pool_get_field(allFields, itr % memberName, char0d, timeLevel) - ! call MPAS_streamAddField(stream % stream, char0d) - ! case (1) - ! call mpas_pool_get_field(allFields, itr % memberName, char1d, timeLevel) - ! call MPAS_streamAddField(stream % stream, char1d) - ! end select - ! end select - - ! if ( threadNum == 0 ) then - ! ! - ! ! Check that stream exists - ! ! - ! nullify(stream) - ! if (.not. MPAS_stream_list_query(manager % streams, streamID, stream, ierr=err_local)) then - ! STREAM_ERROR_WRITE('Requested stream '//trim(streamID)//' does not exist in stream manager') - ! if (present(ierr)) ierr = MPAS_STREAM_MGR_ERROR - ! return - ! end if - - ! ! - ! ! Don't modify an immutable stream - ! ! - ! if (stream % immutable) then - ! STREAM_ERROR_WRITE('Requested stream '//trim(streamID)//' is immutable.') - ! if (present(ierr)) ierr = MPAS_STREAM_MGR_ERROR - ! return - ! end if - - ! ! - ! ! Check that field exists - ! ! - ! info % nDims = -1 - ! call mpas_pool_get_field_info(manager % allFields, fieldName, info) - ! if (info % nDims == -1) then - ! STREAM_ERROR_WRITE('Requested field '//trim(fieldName)//' not available') - ! if (present(ierr)) ierr = MPAS_STREAM_MGR_ERROR - ! return - ! end if - - ! call mpas_log_write('===== my log write stmt .. field is '//trim(fieldName)//' .') - ! ! - - ! end if - - ! end subroutine MPAS_stream_mgr_add_field!}}} - - !----------------------------------------------------------------------- ! routine MPAS_stream_mgr_add_stream_fields @@ -4503,6 +4346,13 @@ end subroutine gen_random call MPAS_streamAddField(stream % stream, real1d) call mpas_log_write('from build_stream: Writing out real1d '//trim(real1d % fieldName)) call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/real1d % isDecomposed/)) + call mpas_log_write('from build_stream: Dimname '//trim(real2d % dimNames(iDim))//' Size $i',intArgs=(/real2d % dimSizes(iDim)/)) + if ( is_decomposed_dim(real1d % dimNames(iDim))) then + size_field = size_field * global_dim_size(real2d % block, real1d % dimNames(iDim)) + else + size_field = size_field * real1d % dimSizes(iDim) + call mpas_log_write('from build_stream ----- Dimname is not decomposed '//trim(real1d % dimNames(iDim))//' Size $i',intArgs=(/real1d % dimSizes(iDim)/)) + end if case (2) call mpas_pool_get_field(allFields, itr % memberName, real2d, timeLevel) call MPAS_streamAddField(stream % stream, real2d) @@ -4510,16 +4360,12 @@ end subroutine gen_random call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/real2d % isDecomposed/)) do iDim = 1, 2 - loc_dim_size = real2d % dimSizes(iDim) call mpas_log_write('from build_stream: Dimname '//trim(real2d % dimNames(iDim))//' Size $i',intArgs=(/real2d % dimSizes(iDim)/)) if ( is_decomposed_dim(real2d % dimNames(iDim))) then - !call mpas_dmpar_sum_int(real2d % block % domain % dminfo, loc_dim_size, global_dim_size) size_field = size_field * global_dim_size(real2d % block, real2d % dimNames(iDim)) - !size_field = size_field * global_dim_size - else - size_field = size_field * loc_dim_size - call mpas_log_write('from build_stream ----- Dimname is not decomposed '//trim(real2d % dimNames(iDim))//' Size $i',intArgs=(/loc_dim_size/)) + size_field = size_field * real2d % dimSizes(iDim) + call mpas_log_write('from build_stream ----- Dimname is not decomposed '//trim(real2d % dimNames(iDim))//' Size $i',intArgs=(/real2d % dimSizes(iDim)/)) end if end do call mpas_log_write('from build_stream ----- cum size is $i',intArgs=(/size_field/)) @@ -4530,6 +4376,12 @@ end subroutine gen_random call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/real3d % isDecomposed/)) do iDim = 1, 3 call mpas_log_write('from build_stream: Dimname '//trim(real3d % dimNames(iDim))//' Size $i',intArgs=(/real3d % dimSizes(iDim)/)) + if ( is_decomposed_dim(real3d % dimNames(iDim))) then + size_field = size_field * global_dim_size(real3d % block, real3d % dimNames(iDim)) + else + size_field = size_field * real3d % dimSizes(iDim) + call mpas_log_write('from build_stream ----- Dimname is not decomposed '//trim(real3d % dimNames(iDim))//' Size $i',intArgs=(/real3d % dimSizes(iDim)/)) + end if end do case (4) call mpas_pool_get_field(allFields, itr % memberName, real4d, timeLevel) @@ -4538,12 +4390,27 @@ end subroutine gen_random call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/real4d % isDecomposed/)) do iDim = 1, 4 call mpas_log_write('from build_stream: Dimname '//trim(real4d % dimNames(iDim))//' Size $i',intArgs=(/real4d % dimSizes(iDim)/)) + if ( is_decomposed_dim(real4d % dimNames(iDim))) then + size_field = size_field * global_dim_size(real4d % block, real4d % dimNames(iDim)) + else + size_field = size_field * real4d % dimSizes(iDim) + call mpas_log_write('from build_stream ----- Dimname is not decomposed '//trim(real4d % dimNames(iDim))//' Size $i',intArgs=(/real4d % dimSizes(iDim)/)) + end if end do case (5) call mpas_pool_get_field(allFields, itr % memberName, real5d, timeLevel) call MPAS_streamAddField(stream % stream, real5d) call mpas_log_write('from build_stream: Writing out real5d '//trim(real5d % fieldName)) call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/real5d % isDecomposed/)) + do iDim = 1, 5 + call mpas_log_write('from build_stream: Dimname '//trim(real5d % dimNames(iDim))//' Size $i',intArgs=(/real5d % dimSizes(iDim)/)) + if ( is_decomposed_dim(real5d % dimNames(iDim))) then + size_field = size_field * global_dim_size(real5d % block, real5d % dimNames(iDim)) + else + size_field = size_field * real5d % dimSizes(iDim) + call mpas_log_write('from build_stream ----- Dimname is not decomposed '//trim(real5d % dimNames(iDim))//' Size $i',intArgs=(/real5d % dimSizes(iDim)/)) + end if + end do end select case (MPAS_POOL_INTEGER) select case (info % nDims) From 2d36c221701d0004fd5313bc19426a1ff8d2d21b Mon Sep 17 00:00:00 2001 From: Abishek Gopal Date: Mon, 22 Jul 2024 11:33:14 -0600 Subject: [PATCH 13/19] more cleanup --- src/framework/mpas_io_streams.F | 16 ++++++++++++---- src/framework/mpas_stream_manager.F | 6 +----- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/framework/mpas_io_streams.F b/src/framework/mpas_io_streams.F index f48b2d1c48..7d59c10f11 100644 --- a/src/framework/mpas_io_streams.F +++ b/src/framework/mpas_io_streams.F @@ -3335,9 +3335,9 @@ subroutine MPAS_writeStream(stream, frame, ierr) if (field_cursor % field_type == FIELD_0D_INT) then - call mpas_log_write('Writing out field '//trim(field_cursor % int0dField % fieldName)) - call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/field_cursor % isDecomposed/)) - call mpas_log_write(' > outer dimension size $i', intArgs=(/field_cursor % totalDimSize/)) + ! call mpas_log_write('Writing out field '//trim(field_cursor % int0dField % fieldName)) + ! call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/field_cursor % isDecomposed/)) + ! call mpas_log_write(' > outer dimension size $i', intArgs=(/field_cursor % totalDimSize/)) !call mpas_log_write('Copying field from first block') int0d_temp = field_cursor % int0dField % scalar @@ -3543,6 +3543,9 @@ subroutine MPAS_writeStream(stream, frame, ierr) else if (field_cursor % field_type == FIELD_0D_REAL) then + !call mpas_log_write('Writing out field '//trim(field_cursor % real0dField % fieldName)) + !call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/field_cursor % isDecomposed/)) + !call mpas_log_write(' > outer dimension size $i', intArgs=(/field_cursor % totalDimSize/)) !call mpas_log_write('Copying field from first block') real0d_temp = field_cursor % real0dField % scalar @@ -3834,7 +3837,6 @@ subroutine MPAS_writeStream(stream, frame, ierr) do j=1,ncons if (field_cursor % isDecomposed) then - ! Gather field from across multiple blocks field_5dreal_ptr => field_cursor % real5dField i = 1 @@ -3889,6 +3891,9 @@ subroutine MPAS_writeStream(stream, frame, ierr) else if (field_cursor % field_type == FIELD_0D_CHAR) then + !call mpas_log_write('Writing out field '//trim(field_cursor % char0dField % fieldName)) + !call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/field_cursor % isDecomposed/)) + !call mpas_log_write(' > outer dimension size $i', intArgs=(/field_cursor % totalDimSize/)) !call mpas_log_write('Copying field from first block') !call mpas_log_write('MGD calling MPAS_io_put_var now...') @@ -3898,6 +3903,9 @@ subroutine MPAS_writeStream(stream, frame, ierr) else if (field_cursor % field_type == FIELD_1D_CHAR) then + !call mpas_log_write('Writing out field '//trim(field_cursor % char1dField % fieldName)) + !call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/field_cursor % isDecomposed/)) + !call mpas_log_write(' > outer dimension size $i', intArgs=(/field_cursor % totalDimSize/)) !call mpas_log_write('Copying field from first block') !call mpas_log_write('MGD calling MPAS_io_put_var now...') diff --git a/src/framework/mpas_stream_manager.F b/src/framework/mpas_stream_manager.F index 630eee1a71..1f361e416e 100644 --- a/src/framework/mpas_stream_manager.F +++ b/src/framework/mpas_stream_manager.F @@ -888,7 +888,6 @@ subroutine MPAS_stream_mgr_add_field(manager, streamID, fieldName, packages, ier return end if - call mpas_log_write('===== my log write stmt .. field is '//trim(fieldName)//' .') ! ! Check that the field does not already exist in the stream ! @@ -4242,8 +4241,7 @@ end subroutine gen_random logical :: active_field integer :: err_level integer :: iDim - !integer(kind=I8KIND) :: size_field, loc_dim_size, global_dim_size - integer :: size_field, loc_dim_size + integer :: size_field if (direction == MPAS_STREAM_OUTPUT) then @@ -4297,8 +4295,6 @@ end subroutine gen_random call mpas_pool_begin_iteration(stream % field_pool) - - FIELD_LOOP: do while ( mpas_pool_get_next_member(stream % field_pool, itr) ) if (itr % memberType == MPAS_POOL_CONFIG) then From 54e6776f953fc8d336b679b35e1bb0710c93eda7 Mon Sep 17 00:00:00 2001 From: Abishek Gopal Date: Tue, 23 Jul 2024 17:40:47 -0600 Subject: [PATCH 14/19] introducing a subroutine to check max field size (bytes) --- src/framework/mpas_stream_manager.F | 221 +++++++++++++++++++--------- 1 file changed, 150 insertions(+), 71 deletions(-) diff --git a/src/framework/mpas_stream_manager.F b/src/framework/mpas_stream_manager.F index 1f361e416e..f9172f574d 100644 --- a/src/framework/mpas_stream_manager.F +++ b/src/framework/mpas_stream_manager.F @@ -3182,6 +3182,7 @@ subroutine write_stream(manager, stream, blockID, timeLevel, mgLevel, forceWrite ! ! Build stream from pools of fields and attributes ! + call check_max_var_size(stream % field_pool, manager % allFields, timeLevel, mgLevel, local_ierr) allocate(stream % stream) call MPAS_createStream(stream % stream, manager % ioContext, stream % filename, stream % io_type, MPAS_IO_WRITE, & precision=stream % precision, clobberRecords=clobberRecords, & @@ -4340,73 +4341,18 @@ end subroutine gen_random case (1) call mpas_pool_get_field(allFields, itr % memberName, real1d, timeLevel) call MPAS_streamAddField(stream % stream, real1d) - call mpas_log_write('from build_stream: Writing out real1d '//trim(real1d % fieldName)) - call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/real1d % isDecomposed/)) - call mpas_log_write('from build_stream: Dimname '//trim(real2d % dimNames(iDim))//' Size $i',intArgs=(/real2d % dimSizes(iDim)/)) - if ( is_decomposed_dim(real1d % dimNames(iDim))) then - size_field = size_field * global_dim_size(real2d % block, real1d % dimNames(iDim)) - else - size_field = size_field * real1d % dimSizes(iDim) - call mpas_log_write('from build_stream ----- Dimname is not decomposed '//trim(real1d % dimNames(iDim))//' Size $i',intArgs=(/real1d % dimSizes(iDim)/)) - end if case (2) call mpas_pool_get_field(allFields, itr % memberName, real2d, timeLevel) call MPAS_streamAddField(stream % stream, real2d) - call mpas_log_write('from build_stream: Writing out real2d '//trim(real2d % fieldName)) - call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/real2d % isDecomposed/)) - - do iDim = 1, 2 - call mpas_log_write('from build_stream: Dimname '//trim(real2d % dimNames(iDim))//' Size $i',intArgs=(/real2d % dimSizes(iDim)/)) - if ( is_decomposed_dim(real2d % dimNames(iDim))) then - size_field = size_field * global_dim_size(real2d % block, real2d % dimNames(iDim)) - else - size_field = size_field * real2d % dimSizes(iDim) - call mpas_log_write('from build_stream ----- Dimname is not decomposed '//trim(real2d % dimNames(iDim))//' Size $i',intArgs=(/real2d % dimSizes(iDim)/)) - end if - end do - call mpas_log_write('from build_stream ----- cum size is $i',intArgs=(/size_field/)) case (3) call mpas_pool_get_field(allFields, itr % memberName, real3d, timeLevel) call MPAS_streamAddField(stream % stream, real3d) - call mpas_log_write('from build_stream: Writing out real3d '//trim(real3d % fieldName)) - call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/real3d % isDecomposed/)) - do iDim = 1, 3 - call mpas_log_write('from build_stream: Dimname '//trim(real3d % dimNames(iDim))//' Size $i',intArgs=(/real3d % dimSizes(iDim)/)) - if ( is_decomposed_dim(real3d % dimNames(iDim))) then - size_field = size_field * global_dim_size(real3d % block, real3d % dimNames(iDim)) - else - size_field = size_field * real3d % dimSizes(iDim) - call mpas_log_write('from build_stream ----- Dimname is not decomposed '//trim(real3d % dimNames(iDim))//' Size $i',intArgs=(/real3d % dimSizes(iDim)/)) - end if - end do case (4) call mpas_pool_get_field(allFields, itr % memberName, real4d, timeLevel) call MPAS_streamAddField(stream % stream, real4d) - call mpas_log_write('from build_stream: Writing out real4d '//trim(real4d % fieldName)) - call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/real4d % isDecomposed/)) - do iDim = 1, 4 - call mpas_log_write('from build_stream: Dimname '//trim(real4d % dimNames(iDim))//' Size $i',intArgs=(/real4d % dimSizes(iDim)/)) - if ( is_decomposed_dim(real4d % dimNames(iDim))) then - size_field = size_field * global_dim_size(real4d % block, real4d % dimNames(iDim)) - else - size_field = size_field * real4d % dimSizes(iDim) - call mpas_log_write('from build_stream ----- Dimname is not decomposed '//trim(real4d % dimNames(iDim))//' Size $i',intArgs=(/real4d % dimSizes(iDim)/)) - end if - end do case (5) call mpas_pool_get_field(allFields, itr % memberName, real5d, timeLevel) call MPAS_streamAddField(stream % stream, real5d) - call mpas_log_write('from build_stream: Writing out real5d '//trim(real5d % fieldName)) - call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/real5d % isDecomposed/)) - do iDim = 1, 5 - call mpas_log_write('from build_stream: Dimname '//trim(real5d % dimNames(iDim))//' Size $i',intArgs=(/real5d % dimSizes(iDim)/)) - if ( is_decomposed_dim(real5d % dimNames(iDim))) then - size_field = size_field * global_dim_size(real5d % block, real5d % dimNames(iDim)) - else - size_field = size_field * real5d % dimSizes(iDim) - call mpas_log_write('from build_stream ----- Dimname is not decomposed '//trim(real5d % dimNames(iDim))//' Size $i',intArgs=(/real5d % dimSizes(iDim)/)) - end if - end do end select case (MPAS_POOL_INTEGER) select case (info % nDims) @@ -4564,6 +4510,117 @@ subroutine update_stream(stream, allFields, timeLevelIn, mgLevelIn, ierr) !{{{ end subroutine update_stream !}}} + subroutine check_max_var_size(field_pool, allFields, timeLevelIn, mgLevelIn, ierr) !{{{ + use iso_c_binding, only : c_sizeof + + implicit none + + type (mpas_pool_type), intent(inout) :: field_pool + type (MPAS_Pool_type), intent(in) :: allFields + integer, intent(in) :: timeLevelIn + integer, intent(in) :: mgLevelIn + integer, intent(out) :: ierr + + type (MPAS_Pool_iterator_type) :: itr + type (mpas_pool_field_info_type) :: info + integer :: timeLevel + + type (field5DReal), pointer :: real5d + type (field4DReal), pointer :: real4d + type (field3DReal), pointer :: real3d + type (field2DReal), pointer :: real2d + type (field1DReal), pointer :: real1d + type (field0DReal), pointer :: real0d + + type (field3DInteger), pointer :: int3d + type (field2DInteger), pointer :: int2d + type (field1DInteger), pointer :: int1d + type (field0DInteger), pointer :: int0d + + type (field1DChar), pointer :: char1d + type (field0DChar), pointer :: char0d + + integer :: max_field_size, field_size + integer :: tmp_int + real :: tmp_real + character :: tmp_char + integer(kind=I8KIND) :: bytes_per_kind, field_bytes + integer(kind=I8KIND), parameter :: int_size = 4_I8KIND + integer(kind=I8KIND), parameter :: real_size = 8_I8KIND + character(len=StrKIND):: message + + ierr = MPAS_STREAM_MGR_NOERR + call mpas_pool_begin_iteration(field_pool) + + do while ( mpas_pool_get_next_member(field_pool, itr) ) + + if (itr % memberType == MPAS_POOL_CONFIG) then + + ! To avoid accidentally matching in case statements below... + info % fieldType = -1 + call mpas_pool_get_field_info(allFields, itr % memberName, info) + ! Set time level to read + if (info % nTimeLevels >= timeLevelIn) then + timeLevel = timeLevelIn + else + timeLevel = 1 + end if + call mpas_log_write('In check_max_var_size, field '//trim(itr % memberName)//' ndims: $i',intArgs=(/info % nDims/)) + select case (info % fieldType) + case (MPAS_POOL_REAL) + select case (info % nDims) + case (0) + field_size = 1 + case (1) + call mpas_pool_get_field(allFields, itr % memberName, real1d, timeLevel) + field_size = global_dim_size(real1d % block, real1d % dimNames, real1d % dimSizes, info % nDims) + case (2) + call mpas_pool_get_field(allFields, itr % memberName, real2d, timeLevel) + field_size = global_dim_size(real2d % block, real2d % dimNames, real2d % dimSizes, info % nDims) + case (3) + call mpas_pool_get_field(allFields, itr % memberName, real3d, timeLevel) + field_size = global_dim_size(real3d % block, real3d % dimNames, real3d % dimSizes, info % nDims) + case (4) + call mpas_pool_get_field(allFields, itr % memberName, real4d, timeLevel) + field_size = global_dim_size(real4d % block, real4d % dimNames, real4d % dimSizes, info % nDims) + case (5) + call mpas_pool_get_field(allFields, itr % memberName, real5d, timeLevel) + field_size = global_dim_size(real5d % block, real5d % dimNames, real5d % dimSizes, info % nDims) + end select + field_bytes = int(field_size, kind=I8KIND) * real_size + case (MPAS_POOL_INTEGER) + select case (info % nDims) + case (0) + field_size = 1 + case (1) + call mpas_pool_get_field(allFields, itr % memberName, int1d, timeLevel) + field_size = global_dim_size(int1d % block, int1d % dimNames, int1d % dimSizes, info % nDims) + case (2) + call mpas_pool_get_field(allFields, itr % memberName, int2d, timeLevel) + field_size = global_dim_size(int2d % block, int2d % dimNames, int2d % dimSizes, info % nDims) + case (3) + call mpas_pool_get_field(allFields, itr % memberName, int3d, timeLevel) + field_size = global_dim_size(int3d % block, int3d % dimNames, int3d % dimSizes, info % nDims) + end select + field_bytes = int(field_size, kind=I8KIND) * int_size + case (MPAS_POOL_CHARACTER) + select case (info % nDims) + case (0) + field_size = 1 + case (1) +! call mpas_pool_get_field(allFields, itr % memberName, char1d, timeLevel) + call mpas_log_write('In check_max_var_size, unsupported type field1DChar.', MPAS_LOG_ERR) + end select + field_bytes = int(field_size, kind=I8KIND) * int_size + end select + write(message,fmt='(A,i12)') 'check_max_var_size.. field_bytes =',field_bytes + call mpas_log_write(message) + end if + end do + + end subroutine check_max_var_size !}}} + + !----------------------------------------------------------------------- ! routine stream_active_pkg_check ! @@ -4916,26 +4973,48 @@ end function is_decomposed_dim !}}} - integer function global_dim_size(block, dimName) !{{{ + integer function global_dim_size(block, dimNames, dimSizes, nDim) !{{{ + use iso_c_binding, only : c_sizeof implicit none - character(len=*), intent(in) :: dimName + character(len=*), intent(in), dimension(:) :: dimNames + integer, intent(in), dimension(:) :: dimSizes type(block_type), intent(in) :: block - integer, pointer :: loc_dim_size - - if (trim(dimName) == 'nCells') then - call mpas_pool_get_dimension(block % dimensions, 'nCellsSolve', loc_dim_size) - else if (trim(dimName) == 'nEdges') then - call mpas_pool_get_dimension(block % dimensions, 'nEdgesSolve', loc_dim_size) - else if (trim(dimName) == 'nVertices') then - call mpas_pool_get_dimension(block % dimensions, 'nVerticesSolve', loc_dim_size) - else - global_dim_size = -1 - end if - - call mpas_dmpar_sum_int(block % domain % dminfo, loc_dim_size, global_dim_size) - call mpas_log_write('from build_stream ----- Dimname is decomposed '//trim(dimName)//' local size $i global size $i ',intArgs=(/loc_dim_size, global_dim_size/)) + integer, intent(in) :: nDim + integer, pointer :: block_dim_size + integer :: iDim, sum_block_dim_size + character(len=StrKIND):: message + + call mpas_log_write('from global_dim_size') + global_dim_size = 1 + write(message,fmt='(A,i12)') 'from global_dim_size.. c_sizeof() =',c_sizeof(iDim) + call mpas_log_write(message) + do iDim = 1, nDim + call mpas_log_write('from global_dim_size: Dimname '//trim(dimNames(iDim))//' Size $i',intArgs=(/dimSizes(iDim)/)) + if ( is_decomposed_dim(dimNames(iDim))) then + if (trim(dimNames(iDim)) == 'nCells') then + call mpas_pool_get_dimension(block % dimensions, 'nCellsSolve', block_dim_size) + else if (trim(dimNames(iDim)) == 'nEdges') then + call mpas_pool_get_dimension(block % dimensions, 'nEdgesSolve', block_dim_size) + else if (trim(dimNames(iDim)) == 'nVertices') then + call mpas_pool_get_dimension(block % dimensions, 'nVerticesSolve', block_dim_size) + else + global_dim_size = -1 + end if + + call mpas_dmpar_sum_int(block % domain % dminfo, block_dim_size, sum_block_dim_size) + call mpas_log_write('from global_dim_size ----- Dimname is decomposed '//trim(dimNames(iDim))//' local size $i global size $i ',intArgs=(/block_dim_size, sum_block_dim_size/)) + + global_dim_size = global_dim_size * sum_block_dim_size + else + global_dim_size = global_dim_size * dimSizes(iDim) + call mpas_log_write('from global_dim_size ----- Dimname is not decomposed '//trim(dimNames(iDim))//' Size $i',intArgs=(/dimSizes(iDim)/)) + end if + end do + + call mpas_log_write('from global_dim_size: cumulative global_dim_size $i',intArgs=(/global_dim_size/)) + end function global_dim_size !}}} From a12be08ee3b87d69c91b3ba5efa0fda5ed45a7c6 Mon Sep 17 00:00:00 2001 From: Abishek Gopal Date: Wed, 24 Jul 2024 16:39:42 -0600 Subject: [PATCH 15/19] computing max field size in bytes, and comparing with max allowed for cdf2 --- src/framework/mpas_stream_manager.F | 40 +++++++++++++++-------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/src/framework/mpas_stream_manager.F b/src/framework/mpas_stream_manager.F index f9172f574d..df99a81892 100644 --- a/src/framework/mpas_stream_manager.F +++ b/src/framework/mpas_stream_manager.F @@ -3084,7 +3084,7 @@ subroutine write_stream(manager, stream, blockID, timeLevel, mgLevel, forceWrite character (len=StrKIND) :: now_string, time_string character (len=StrKIND) :: temp_filename, actualWhen character (len=StrKIND) :: err_string - logical :: ringing_alarm, recordSeek, swapRecords + logical :: ringing_alarm, recordSeek, swapRecords, use_cdf2 logical :: clobberRecords, clobberFiles, truncateFiles integer :: maxRecords, tempRecord integer :: local_ierr, threadNum @@ -3182,7 +3182,7 @@ subroutine write_stream(manager, stream, blockID, timeLevel, mgLevel, forceWrite ! ! Build stream from pools of fields and attributes ! - call check_max_var_size(stream % field_pool, manager % allFields, timeLevel, mgLevel, local_ierr) + call check_max_var_size(stream % field_pool, manager % allFields, timeLevel, mgLevel, use_cdf2) allocate(stream % stream) call MPAS_createStream(stream % stream, manager % ioContext, stream % filename, stream % io_type, MPAS_IO_WRITE, & precision=stream % precision, clobberRecords=clobberRecords, & @@ -4510,7 +4510,7 @@ subroutine update_stream(stream, allFields, timeLevelIn, mgLevelIn, ierr) !{{{ end subroutine update_stream !}}} - subroutine check_max_var_size(field_pool, allFields, timeLevelIn, mgLevelIn, ierr) !{{{ + subroutine check_max_var_size(field_pool, allFields, timeLevelIn, mgLevelIn, use_cdf2) !{{{ use iso_c_binding, only : c_sizeof implicit none @@ -4519,7 +4519,7 @@ subroutine check_max_var_size(field_pool, allFields, timeLevelIn, mgLevelIn, ier type (MPAS_Pool_type), intent(in) :: allFields integer, intent(in) :: timeLevelIn integer, intent(in) :: mgLevelIn - integer, intent(out) :: ierr + logical, intent(out) :: use_cdf2 type (MPAS_Pool_iterator_type) :: itr type (mpas_pool_field_info_type) :: info @@ -4542,16 +4542,18 @@ subroutine check_max_var_size(field_pool, allFields, timeLevelIn, mgLevelIn, ier integer :: max_field_size, field_size integer :: tmp_int - real :: tmp_real + real(kind=RKIND) :: tmp_real character :: tmp_char - integer(kind=I8KIND) :: bytes_per_kind, field_bytes - integer(kind=I8KIND), parameter :: int_size = 4_I8KIND - integer(kind=I8KIND), parameter :: real_size = 8_I8KIND + integer(kind=I8KIND) :: max_field_bytes, field_bytes + integer(kind=I8KIND), parameter :: int_size = c_sizeof(tmp_int) + integer(kind=I8KIND), parameter :: real_size = c_sizeof(tmp_real) + integer(kind=I8KIND), parameter :: char_size = c_sizeof(tmp_char) + integer(kind=I8KIND), parameter :: cdf2_max_bytes = 2**32 - 4 character(len=StrKIND):: message - ierr = MPAS_STREAM_MGR_NOERR call mpas_pool_begin_iteration(field_pool) - + max_field_bytes = 0 + field_bytes = 0 do while ( mpas_pool_get_next_member(field_pool, itr) ) if (itr % memberType == MPAS_POOL_CONFIG) then @@ -4611,12 +4613,16 @@ subroutine check_max_var_size(field_pool, allFields, timeLevelIn, mgLevelIn, ier ! call mpas_pool_get_field(allFields, itr % memberName, char1d, timeLevel) call mpas_log_write('In check_max_var_size, unsupported type field1DChar.', MPAS_LOG_ERR) end select - field_bytes = int(field_size, kind=I8KIND) * int_size + field_bytes = int(field_size, kind=I8KIND) * char_size end select - write(message,fmt='(A,i12)') 'check_max_var_size.. field_bytes =',field_bytes + max_field_bytes = merge(field_bytes, max_field_bytes, field_bytes > max_field_bytes) + write(message,fmt='(A,i14,A,i18,A,i18)') 'check_max_var_size.. field_bytes =',field_bytes,' max_field_bytes =',max_field_bytes, & + & ' cdf2_max_bytes =',cdf2_max_bytes call mpas_log_write(message) end if end do + use_cdf2 = cdf2_max_bytes > max_field_bytes + call mpas_log_write('In check_max_var_size use_cdf2: $l',logicArgs=(/use_cdf2/)) end subroutine check_max_var_size !}}} @@ -4986,12 +4992,8 @@ integer function global_dim_size(block, dimNames, dimSizes, nDim) !{{{ integer :: iDim, sum_block_dim_size character(len=StrKIND):: message - call mpas_log_write('from global_dim_size') global_dim_size = 1 - write(message,fmt='(A,i12)') 'from global_dim_size.. c_sizeof() =',c_sizeof(iDim) - call mpas_log_write(message) do iDim = 1, nDim - call mpas_log_write('from global_dim_size: Dimname '//trim(dimNames(iDim))//' Size $i',intArgs=(/dimSizes(iDim)/)) if ( is_decomposed_dim(dimNames(iDim))) then if (trim(dimNames(iDim)) == 'nCells') then call mpas_pool_get_dimension(block % dimensions, 'nCellsSolve', block_dim_size) @@ -5004,16 +5006,16 @@ integer function global_dim_size(block, dimNames, dimSizes, nDim) !{{{ end if call mpas_dmpar_sum_int(block % domain % dminfo, block_dim_size, sum_block_dim_size) - call mpas_log_write('from global_dim_size ----- Dimname is decomposed '//trim(dimNames(iDim))//' local size $i global size $i ',intArgs=(/block_dim_size, sum_block_dim_size/)) + call mpas_log_write('----- from global_dim_size ----- Dimname is decomposed '//trim(dimNames(iDim))//' local size $i global size $i ',intArgs=(/block_dim_size, sum_block_dim_size/)) global_dim_size = global_dim_size * sum_block_dim_size else global_dim_size = global_dim_size * dimSizes(iDim) - call mpas_log_write('from global_dim_size ----- Dimname is not decomposed '//trim(dimNames(iDim))//' Size $i',intArgs=(/dimSizes(iDim)/)) + call mpas_log_write('----- from global_dim_size ----- Dimname is not decomposed '//trim(dimNames(iDim))//' Size $i',intArgs=(/dimSizes(iDim)/)) end if end do - call mpas_log_write('from global_dim_size: cumulative global_dim_size $i',intArgs=(/global_dim_size/)) + call mpas_log_write('----- from global_dim_size: cumulative global_dim_size $i',intArgs=(/global_dim_size/)) end function global_dim_size !}}} From cd764aab379f93ade1fd81b3753f89f8160f44cf Mon Sep 17 00:00:00 2001 From: Abishek Gopal Date: Thu, 25 Jul 2024 12:11:23 -0600 Subject: [PATCH 16/19] cleaning up build_stream --- src/framework/mpas_stream_manager.F | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/src/framework/mpas_stream_manager.F b/src/framework/mpas_stream_manager.F index df99a81892..4fd727db6a 100644 --- a/src/framework/mpas_stream_manager.F +++ b/src/framework/mpas_stream_manager.F @@ -4241,8 +4241,6 @@ end subroutine gen_random character (len=StrKIND), pointer :: packages logical :: active_field integer :: err_level - integer :: iDim - integer :: size_field if (direction == MPAS_STREAM_OUTPUT) then @@ -4336,8 +4334,6 @@ end subroutine gen_random case (0) call mpas_pool_get_field(allFields, itr % memberName, real0d, timeLevel) call MPAS_streamAddField(stream % stream, real0d) - call mpas_log_write('from build_stream: Writing out real0d '//trim(real0d % fieldName)) - call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/real0d % isDecomposed/)) case (1) call mpas_pool_get_field(allFields, itr % memberName, real1d, timeLevel) call MPAS_streamAddField(stream % stream, real1d) @@ -4359,23 +4355,15 @@ end subroutine gen_random case (0) call mpas_pool_get_field(allFields, itr % memberName, int0d, timeLevel) call MPAS_streamAddField(stream % stream, int0d) - call mpas_log_write('from build_stream: Writing out int0d '//trim(int0d % fieldName)) - call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/int0d % isDecomposed/)) case (1) call mpas_pool_get_field(allFields, itr % memberName, int1d, timeLevel) call MPAS_streamAddField(stream % stream, int1d) - call mpas_log_write('from build_stream: Writing out int1d '//trim(int1d % fieldName)) - call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/int1d % isDecomposed/)) case (2) call mpas_pool_get_field(allFields, itr % memberName, int2d, timeLevel) call MPAS_streamAddField(stream % stream, int2d) - call mpas_log_write('from build_stream: Writing out int2d '//trim(int2d % fieldName)) - call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/int2d % isDecomposed/)) case (3) call mpas_pool_get_field(allFields, itr % memberName, int3d, timeLevel) call MPAS_streamAddField(stream % stream, int3d) - call mpas_log_write('from build_stream: Writing out int3d '//trim(int3d % fieldName)) - call mpas_log_write(' > is the field decomposed? $l', logicArgs=(/int3d % isDecomposed/)) end select case (MPAS_POOL_CHARACTER) select case (info % nDims) From 04e2291f3d59f2b9d1c9eff64f99ebfd04af428f Mon Sep 17 00:00:00 2001 From: Abishek Gopal Date: Thu, 25 Jul 2024 13:29:19 -0600 Subject: [PATCH 17/19] fix --- src/framework/mpas_stream_manager.F | 1 - 1 file changed, 1 deletion(-) diff --git a/src/framework/mpas_stream_manager.F b/src/framework/mpas_stream_manager.F index 4fd727db6a..a24308bf83 100644 --- a/src/framework/mpas_stream_manager.F +++ b/src/framework/mpas_stream_manager.F @@ -4326,7 +4326,6 @@ end subroutine gen_random timeLevel = 1 end if - size_field = 1 select case (info % fieldType) case (MPAS_POOL_REAL) From cbcb2365881356c9781a928ef7102ca3cf59cec5 Mon Sep 17 00:00:00 2001 From: Abishek Gopal Date: Mon, 29 Jul 2024 08:48:12 -0600 Subject: [PATCH 18/19] Addressing most review comments --- src/framework/mpas_stream_manager.F | 132 ++++++++++++++-------------- 1 file changed, 65 insertions(+), 67 deletions(-) diff --git a/src/framework/mpas_stream_manager.F b/src/framework/mpas_stream_manager.F index a24308bf83..b644b2e805 100644 --- a/src/framework/mpas_stream_manager.F +++ b/src/framework/mpas_stream_manager.F @@ -3084,10 +3084,12 @@ subroutine write_stream(manager, stream, blockID, timeLevel, mgLevel, forceWrite character (len=StrKIND) :: now_string, time_string character (len=StrKIND) :: temp_filename, actualWhen character (len=StrKIND) :: err_string - logical :: ringing_alarm, recordSeek, swapRecords, use_cdf2 + logical :: ringing_alarm, recordSeek, swapRecords logical :: clobberRecords, clobberFiles, truncateFiles integer :: maxRecords, tempRecord integer :: local_ierr, threadNum + integer(kind=I8KIND) :: max_var_size_bytes + character(len=StrKIND):: message threadNum = mpas_threading_get_thread_num() @@ -3182,7 +3184,9 @@ subroutine write_stream(manager, stream, blockID, timeLevel, mgLevel, forceWrite ! ! Build stream from pools of fields and attributes ! - call check_max_var_size(stream % field_pool, manager % allFields, timeLevel, mgLevel, use_cdf2) + max_var_size_bytes = stream_max_var_size(stream % field_pool, manager % allFields) + write(message,fmt='(A,i18)') 'final max_var_size_bytes =',max_var_size_bytes + call mpas_log_write(message) allocate(stream % stream) call MPAS_createStream(stream % stream, manager % ioContext, stream % filename, stream % io_type, MPAS_IO_WRITE, & precision=stream % precision, clobberRecords=clobberRecords, & @@ -4497,16 +4501,13 @@ subroutine update_stream(stream, allFields, timeLevelIn, mgLevelIn, ierr) !{{{ end subroutine update_stream !}}} - subroutine check_max_var_size(field_pool, allFields, timeLevelIn, mgLevelIn, use_cdf2) !{{{ + integer(kind=I8KIND) function stream_max_var_size(field_pool, allFields) !{{{ use iso_c_binding, only : c_sizeof implicit none type (mpas_pool_type), intent(inout) :: field_pool type (MPAS_Pool_type), intent(in) :: allFields - integer, intent(in) :: timeLevelIn - integer, intent(in) :: mgLevelIn - logical, intent(out) :: use_cdf2 type (MPAS_Pool_iterator_type) :: itr type (mpas_pool_field_info_type) :: info @@ -4528,18 +4529,15 @@ subroutine check_max_var_size(field_pool, allFields, timeLevelIn, mgLevelIn, use type (field0DChar), pointer :: char0d integer :: max_field_size, field_size - integer :: tmp_int - real(kind=RKIND) :: tmp_real character :: tmp_char - integer(kind=I8KIND) :: max_field_bytes, field_bytes - integer(kind=I8KIND), parameter :: int_size = c_sizeof(tmp_int) - integer(kind=I8KIND), parameter :: real_size = c_sizeof(tmp_real) + integer(kind=I8KIND) :: field_bytes + integer(kind=I8KIND), parameter :: int_size = c_sizeof(1) + integer(kind=I8KIND), parameter :: real_size = c_sizeof(1.0_RKIND) integer(kind=I8KIND), parameter :: char_size = c_sizeof(tmp_char) - integer(kind=I8KIND), parameter :: cdf2_max_bytes = 2**32 - 4 character(len=StrKIND):: message call mpas_pool_begin_iteration(field_pool) - max_field_bytes = 0 + stream_max_var_size = 0 field_bytes = 0 do while ( mpas_pool_get_next_member(field_pool, itr) ) @@ -4548,12 +4546,8 @@ subroutine check_max_var_size(field_pool, allFields, timeLevelIn, mgLevelIn, use ! To avoid accidentally matching in case statements below... info % fieldType = -1 call mpas_pool_get_field_info(allFields, itr % memberName, info) - ! Set time level to read - if (info % nTimeLevels >= timeLevelIn) then - timeLevel = timeLevelIn - else - timeLevel = 1 - end if + ! Reading first time level + timeLevel = 1 call mpas_log_write('In check_max_var_size, field '//trim(itr % memberName)//' ndims: $i',intArgs=(/info % nDims/)) select case (info % fieldType) case (MPAS_POOL_REAL) @@ -4562,19 +4556,21 @@ subroutine check_max_var_size(field_pool, allFields, timeLevelIn, mgLevelIn, use field_size = 1 case (1) call mpas_pool_get_field(allFields, itr % memberName, real1d, timeLevel) - field_size = global_dim_size(real1d % block, real1d % dimNames, real1d % dimSizes, info % nDims) + field_size = global_dim_size(real1d % block, real1d % dimNames, real1d % isVarArray) case (2) call mpas_pool_get_field(allFields, itr % memberName, real2d, timeLevel) - field_size = global_dim_size(real2d % block, real2d % dimNames, real2d % dimSizes, info % nDims) + field_size = global_dim_size(real2d % block, real2d % dimNames, real2d % isVarArray) case (3) + call mpas_log_write('before get field') call mpas_pool_get_field(allFields, itr % memberName, real3d, timeLevel) - field_size = global_dim_size(real3d % block, real3d % dimNames, real3d % dimSizes, info % nDims) + call mpas_log_write('after get field') + field_size = global_dim_size(real3d % block, real3d % dimNames, real3d % isVarArray) case (4) call mpas_pool_get_field(allFields, itr % memberName, real4d, timeLevel) - field_size = global_dim_size(real4d % block, real4d % dimNames, real4d % dimSizes, info % nDims) + field_size = global_dim_size(real4d % block, real4d % dimNames, real4d % isVarArray) case (5) call mpas_pool_get_field(allFields, itr % memberName, real5d, timeLevel) - field_size = global_dim_size(real5d % block, real5d % dimNames, real5d % dimSizes, info % nDims) + field_size = global_dim_size(real5d % block, real5d % dimNames, real5d % isVarArray) end select field_bytes = int(field_size, kind=I8KIND) * real_size case (MPAS_POOL_INTEGER) @@ -4583,13 +4579,13 @@ subroutine check_max_var_size(field_pool, allFields, timeLevelIn, mgLevelIn, use field_size = 1 case (1) call mpas_pool_get_field(allFields, itr % memberName, int1d, timeLevel) - field_size = global_dim_size(int1d % block, int1d % dimNames, int1d % dimSizes, info % nDims) + field_size = global_dim_size(int1d % block, int1d % dimNames, int1d % isVarArray) case (2) call mpas_pool_get_field(allFields, itr % memberName, int2d, timeLevel) - field_size = global_dim_size(int2d % block, int2d % dimNames, int2d % dimSizes, info % nDims) + field_size = global_dim_size(int2d % block, int2d % dimNames, int2d % isVarArray) case (3) call mpas_pool_get_field(allFields, itr % memberName, int3d, timeLevel) - field_size = global_dim_size(int3d % block, int3d % dimNames, int3d % dimSizes, info % nDims) + field_size = global_dim_size(int3d % block, int3d % dimNames, int3d % isVarArray) end select field_bytes = int(field_size, kind=I8KIND) * int_size case (MPAS_POOL_CHARACTER) @@ -4602,16 +4598,13 @@ subroutine check_max_var_size(field_pool, allFields, timeLevelIn, mgLevelIn, use end select field_bytes = int(field_size, kind=I8KIND) * char_size end select - max_field_bytes = merge(field_bytes, max_field_bytes, field_bytes > max_field_bytes) - write(message,fmt='(A,i14,A,i18,A,i18)') 'check_max_var_size.. field_bytes =',field_bytes,' max_field_bytes =',max_field_bytes, & - & ' cdf2_max_bytes =',cdf2_max_bytes + stream_max_var_size = merge(field_bytes, stream_max_var_size, field_bytes > stream_max_var_size) + write(message,fmt='(A,i14,A,i18)') 'check_max_var_size.. field_bytes =',field_bytes,' stream_max_var_size =',stream_max_var_size call mpas_log_write(message) end if end do - use_cdf2 = cdf2_max_bytes > max_field_bytes - call mpas_log_write('In check_max_var_size use_cdf2: $l',logicArgs=(/use_cdf2/)) - end subroutine check_max_var_size !}}} + end function stream_max_var_size !}}} !----------------------------------------------------------------------- @@ -4966,46 +4959,51 @@ end function is_decomposed_dim !}}} - integer function global_dim_size(block, dimNames, dimSizes, nDim) !{{{ - use iso_c_binding, only : c_sizeof + integer function global_dim_size(block, dimNames, isVarArray) !{{{ - implicit none + implicit none + + character(len=*), intent(in), dimension(:) :: dimNames + type(block_type), intent(in) :: block + logical, intent(in) :: isVarArray + integer, pointer :: block_dim_size + integer :: iDim, iDimStart, iDimEnd, sum_block_dim_size + character(len=StrKIND):: message - character(len=*), intent(in), dimension(:) :: dimNames - integer, intent(in), dimension(:) :: dimSizes - type(block_type), intent(in) :: block - integer, intent(in) :: nDim - integer, pointer :: block_dim_size - integer :: iDim, sum_block_dim_size - character(len=StrKIND):: message - - global_dim_size = 1 - do iDim = 1, nDim - if ( is_decomposed_dim(dimNames(iDim))) then - if (trim(dimNames(iDim)) == 'nCells') then - call mpas_pool_get_dimension(block % dimensions, 'nCellsSolve', block_dim_size) - else if (trim(dimNames(iDim)) == 'nEdges') then - call mpas_pool_get_dimension(block % dimensions, 'nEdgesSolve', block_dim_size) - else if (trim(dimNames(iDim)) == 'nVertices') then - call mpas_pool_get_dimension(block % dimensions, 'nVerticesSolve', block_dim_size) + call mpas_log_write('----- from global_dim_size: size $i',intArgs=(/size(dimNames)/)) + global_dim_size = 1 + ! Skip left-most dimension, as constituent elements of varArrays are written out separately + iDimStart = merge(2, 1, isVarArray) + iDimEnd = size(dimNames) + call mpas_log_write('----- from global_dim_size: iDimStart $i iDimEnd $i',intArgs=(/iDimStart,iDimEnd/)) + do iDim = iDimStart, iDimEnd + if ( is_decomposed_dim(dimNames(iDim))) then + if (trim(dimNames(iDim)) == 'nCells') then + call mpas_pool_get_dimension(block % dimensions, 'nCellsSolve', block_dim_size) + else if (trim(dimNames(iDim)) == 'nEdges') then + call mpas_pool_get_dimension(block % dimensions, 'nEdgesSolve', block_dim_size) + else if (trim(dimNames(iDim)) == 'nVertices') then + call mpas_pool_get_dimension(block % dimensions, 'nVerticesSolve', block_dim_size) + else + global_dim_size = -1 + end if + + call mpas_dmpar_sum_int(block % domain % dminfo, block_dim_size, sum_block_dim_size) + call mpas_log_write('----- from global_dim_size ----- Dimname is decomposed '//trim(dimNames(iDim))//' local size $i global size $i ',intArgs=(/block_dim_size, sum_block_dim_size/)) + + global_dim_size = global_dim_size * sum_block_dim_size else - global_dim_size = -1 - end if - - call mpas_dmpar_sum_int(block % domain % dminfo, block_dim_size, sum_block_dim_size) - call mpas_log_write('----- from global_dim_size ----- Dimname is decomposed '//trim(dimNames(iDim))//' local size $i global size $i ',intArgs=(/block_dim_size, sum_block_dim_size/)) - - global_dim_size = global_dim_size * sum_block_dim_size - else - global_dim_size = global_dim_size * dimSizes(iDim) - call mpas_log_write('----- from global_dim_size ----- Dimname is not decomposed '//trim(dimNames(iDim))//' Size $i',intArgs=(/dimSizes(iDim)/)) - end if - end do + call mpas_log_write('----- from global_dim_size ----- before get dim... Dimname is not decomposed '//trim(dimNames(iDim))) + call mpas_pool_get_dimension(block % dimensions, dimNames(iDim), block_dim_size) + global_dim_size = global_dim_size * block_dim_size + call mpas_log_write('----- from global_dim_size ----- Dimname is not decomposed '//trim(dimNames(iDim))//' Size $i',intArgs=(/block_dim_size/)) + end if + end do - call mpas_log_write('----- from global_dim_size: cumulative global_dim_size $i',intArgs=(/global_dim_size/)) + call mpas_log_write('----- from global_dim_size: cumulative global_dim_size $i',intArgs=(/global_dim_size/)) -end function global_dim_size !}}} + end function global_dim_size !}}} From cfd894e1f1aa52dc516082e5a9aa87618daac7d0 Mon Sep 17 00:00:00 2001 From: Abishek Gopal Date: Mon, 29 Jul 2024 09:54:14 -0600 Subject: [PATCH 19/19] changing all field sizes to int64 --- src/framework/mpas_stream_manager.F | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/src/framework/mpas_stream_manager.F b/src/framework/mpas_stream_manager.F index b644b2e805..63550f939a 100644 --- a/src/framework/mpas_stream_manager.F +++ b/src/framework/mpas_stream_manager.F @@ -4528,9 +4528,8 @@ integer(kind=I8KIND) function stream_max_var_size(field_pool, allFields) !{{{ type (field1DChar), pointer :: char1d type (field0DChar), pointer :: char0d - integer :: max_field_size, field_size character :: tmp_char - integer(kind=I8KIND) :: field_bytes + integer(kind=I8KIND) :: field_bytes, max_field_size, field_size integer(kind=I8KIND), parameter :: int_size = c_sizeof(1) integer(kind=I8KIND), parameter :: real_size = c_sizeof(1.0_RKIND) integer(kind=I8KIND), parameter :: char_size = c_sizeof(tmp_char) @@ -4587,7 +4586,7 @@ integer(kind=I8KIND) function stream_max_var_size(field_pool, allFields) !{{{ call mpas_pool_get_field(allFields, itr % memberName, int3d, timeLevel) field_size = global_dim_size(int3d % block, int3d % dimNames, int3d % isVarArray) end select - field_bytes = int(field_size, kind=I8KIND) * int_size + field_bytes = field_size * int_size case (MPAS_POOL_CHARACTER) select case (info % nDims) case (0) @@ -4596,7 +4595,7 @@ integer(kind=I8KIND) function stream_max_var_size(field_pool, allFields) !{{{ ! call mpas_pool_get_field(allFields, itr % memberName, char1d, timeLevel) call mpas_log_write('In check_max_var_size, unsupported type field1DChar.', MPAS_LOG_ERR) end select - field_bytes = int(field_size, kind=I8KIND) * char_size + field_bytes = field_size * char_size end select stream_max_var_size = merge(field_bytes, stream_max_var_size, field_bytes > stream_max_var_size) write(message,fmt='(A,i14,A,i18)') 'check_max_var_size.. field_bytes =',field_bytes,' stream_max_var_size =',stream_max_var_size @@ -4959,7 +4958,7 @@ end function is_decomposed_dim !}}} - integer function global_dim_size(block, dimNames, isVarArray) !{{{ + integer(kind=I8KIND) function global_dim_size(block, dimNames, isVarArray) !{{{ implicit none @@ -4967,7 +4966,8 @@ integer function global_dim_size(block, dimNames, isVarArray) !{{{ type(block_type), intent(in) :: block logical, intent(in) :: isVarArray integer, pointer :: block_dim_size - integer :: iDim, iDimStart, iDimEnd, sum_block_dim_size + integer(kind=I8KIND):: sum_block_dim_size, block_dim_size_int8 + integer :: iDim, iDimStart, iDimEnd character(len=StrKIND):: message call mpas_log_write('----- from global_dim_size: size $i',intArgs=(/size(dimNames)/)) @@ -4988,20 +4988,24 @@ integer function global_dim_size(block, dimNames, isVarArray) !{{{ global_dim_size = -1 end if - call mpas_dmpar_sum_int(block % domain % dminfo, block_dim_size, sum_block_dim_size) - call mpas_log_write('----- from global_dim_size ----- Dimname is decomposed '//trim(dimNames(iDim))//' local size $i global size $i ',intArgs=(/block_dim_size, sum_block_dim_size/)) + block_dim_size_int8 = int(block_dim_size, kind=I8KIND) + + call mpas_dmpar_sum_int8(block % domain % dminfo, block_dim_size_int8, sum_block_dim_size) + write(message,fmt='(A,i18,A,i18)') '----- from global_dim_size: Dimname is decomposed '//trim(dimNames(iDim))//' local size =',block_dim_size,' sum_block_dim_size=', sum_block_dim_size + call mpas_log_write(message) global_dim_size = global_dim_size * sum_block_dim_size else call mpas_log_write('----- from global_dim_size ----- before get dim... Dimname is not decomposed '//trim(dimNames(iDim))) call mpas_pool_get_dimension(block % dimensions, dimNames(iDim), block_dim_size) - global_dim_size = global_dim_size * block_dim_size + block_dim_size_int8 = int(block_dim_size, kind=I8KIND) + global_dim_size = global_dim_size * block_dim_size_int8 call mpas_log_write('----- from global_dim_size ----- Dimname is not decomposed '//trim(dimNames(iDim))//' Size $i',intArgs=(/block_dim_size/)) end if end do - call mpas_log_write('----- from global_dim_size: cumulative global_dim_size $i',intArgs=(/global_dim_size/)) - + write(message,fmt='(A,i18)') '----- from global_dim_size: cumulative global_dim_size =',global_dim_size + call mpas_log_write(message) end function global_dim_size !}}}