Unit tests (using pFunit)

.github/workflows/test_suite.yml

@@ -41,19 +41,43 @@ jobs:
           . ftorch/bin/activate
           pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu
 
+      # MPI is required by pFUnit
+      - name: Install an MPI distribution
+        run: |
+          sudo apt update
+          sudo apt install mpich
+
+      - name: Install pFUnit
+        run: |
+          export FC=/usr/bin/gfortran
+          export MPIF90=/usr/bin/mpif90
+          # TODO: Avoid version pinning (needed because version appears in install path)
+          git clone -b v4.10.0 https://github.com/Goddard-Fortran-Ecosystem/pFUnit.git
+          mkdir pFUnit/build
+          cd pFUnit/build
+          cmake -DMPI=YES ..
+          make install
+
       - name: Build FTorch
         run: |
           . ftorch/bin/activate
           VN=$(python -c "import sys; print('.'.join(sys.version.split('.')[:2]))")
           export Torch_DIR=${VIRTUAL_ENV}/lib/python${VN}/site-packages
           export BUILD_DIR=$(pwd)/src/build
+          # TODO: How can we avoid the pFUnit version ending up in the install path?
+          export PFUNIT_DIR=$(pwd)/pFUnit/build/installed/PFUNIT-4.10
           mkdir ${BUILD_DIR}
           cd ${BUILD_DIR}
-          cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${BUILD_DIR} -DCMAKE_BUILD_TESTS=TRUE -DCMAKE_Fortran_FLAGS="-std=${{ matrix.std }}"
+          cmake .. \
+            -DCMAKE_BUILD_TYPE=Release \
+            -DCMAKE_BUILD_TESTS=TRUE \
+            -DCMAKE_PREFIX_PATH="${PFUNIT_DIR}:${Torch_DIR}" \
+            -DCMAKE_INSTALL_PREFIX=${BUILD_DIR} \
+            -DCMAKE_Fortran_FLAGS="-std=${{ matrix.std }}"
           cmake --build .
           cmake --install .
 
       - name: Integration tests
         run: |
           . ftorch/bin/activate
-          ./run_integration_tests.sh -V
+          ./run_test_suite.sh -V

pages/testing.md

@@ -4,23 +4,26 @@ title: FTorch test suite
 
 ## Testing
 
-FTorch's test suite is currently comprised of integration tests based on a
-subset of the [examples](examples.html). These tests are built and run and their
-outputs are analysed to check they contain expected regular expressions.
+FTorch's test suite is currently comprised of unit tests, as well as integration
+tests based on a subset of the [examples](examples.html). These tests are built
+and run and their outputs are analysed to check they contain expected regular
+expressions.
 
-### Building the integration tests
+### Building
 
-To enable FTorch's integration tests, ensure that the `CMAKE_BUILD_TESTS` option
-is set to `TRUE` for the build.
+To enable FTorch's test suite, ensure that the `CMAKE_BUILD_TESTS` option
+is set to `TRUE` for the build. For the unit tests, you will also need to
+install [pFUnit](https://github.com/Goddard-Fortran-Ecosystem/pFUnit) and
+pass its install directory to the `CMAKE_PREFIX_PATH` when building FTorch.
 
-### Running the integration tests
+### Running
 
 Once the build is complete, activate the Python virtual environment you created
 for FTorch<sup>1</sup> and simply run the
 [helper script](https://github.com/Cambridge-ICCS/FTorch/blob/main/run_integration_tests.sh)
 in the root FTorch directory:
 ```
-./run_integration_tests.sh
+./run_test_suite.sh
 ```
 This will automatically install any additional dependencies for the examples.
 
@@ -30,9 +33,9 @@ virtual environment) then either
 the purposes of testing, or note that this script may have your Python
 environment install some modules._
 
-Alternatively, individual tests may be run by going to the corresponding
-subdirectory of `${BUILD_DIR}/test/examples` (where `${BUILD_DIR}` is the build
-directory for FTorch) and calling `ctest`. This will produce a report on which
-tests passed and which failed for your build. Note that some of the examples
-have additional dependencies, which may need installing into your virtual
-environment.
+Alternatively, individual tests may be run by going to `${BUILD_DIR}/test/unit`
+or the corresponding subdirectory of `${BUILD_DIR}/test/examples` (where
+`${BUILD_DIR}` is the build directory for FTorch) and calling `ctest`. This
+will produce a report on which tests passed and which failed for your build.
+Note that some of the examples have additional dependencies, which may need
+installing into your virtual environment.

run_test_suite.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+# ---
+# Execute this shell script to run FTorch's test suite. This includes both unit
+# tests and integration tests.
+#
+# Assumes FTorch has been built with the `-DCMAKE_BUILD_TESTS=TRUE` option.
+# The `BUILD_DIR` variable in this script should be updated as appropriate for
+# your configuration.
+#
+# See `src/test/README.md` for more details on the test suite.
+# ---
+
+set -eu
+
+CTEST_ARGS=$@
+BUILD_DIR=src/build
+
+# Unit tests
+cd ${BUILD_DIR}/test/unit
+ctest ${CTEST_ARGS}
+cd -
+
+# # Integration tests
+# EXAMPLES="1_SimpleNet 2_ResNet18 4_MultiIO 6_Autograd"
+# for EXAMPLE in ${EXAMPLES}; do
+#   pip -q install -r examples/${EXAMPLE}/requirements.txt
+#   cd ${BUILD_DIR}/test/examples/${EXAMPLE}
+#   ctest ${CTEST_ARGS}
+#   cd -
+# done

src/CMakeLists.txt

@@ -87,6 +87,12 @@ install(FILES "${CMAKE_BINARY_DIR}/modules/ftorch_test_utils.mod"
 
 # Build integration tests
 if(CMAKE_BUILD_TESTS)
+  enable_testing()
+
+  # Unit tests
+  add_subdirectory(test/unit)
+
+  # Integration tests
   file(MAKE_DIRECTORY test/examples)
   file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/../examples/CMakeLists.txt
     DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/test/examples

src/test/README.md

@@ -1,31 +1,41 @@
-# Integration tests
+# FTorch test suite
 
-This subdirectory is populated with code and data from the examples directory
-during the test suite. The examples can be run as integration tests to verify
-that FTorch is working as expected in those cases.
+## Unit tests
 
-Currently, only the first ('SimpleNet') and second ('ResNet18') examples have
-been implemented as integration tests.
+The `unit` subdirectory contains unit tests written using
+[pFUnit](https://github.com/Goddard-Fortran-Ecosystem/pFUnit). To be able to
+compile these, you will need to first install pFUnit and then pass its install
+directory to the `CMAKE_PREFIX_PATH` when building FTorch.
+
+
+## Integration tests
+
+The `examples` subdirectory is populated with code and data from the examples
+directory during the test suite. The examples can be run as integration tests to
+verify that FTorch is working as expected in those cases.
+
+Currently, only a subset of the examples have been implemented as integration
+tests.
 
 ## Building
 
-To build them as tests, add the flag
+To build with testing enabled, add the flag
 ```
 -DCMAKE_BUILD_TESTS=TRUE
 ```
 when building FTorch.
 
 ## Running
 
-Having built the integration tests, ensure the Python virtual environment is
-active and run them by going to the corresponding subdirectory of
-`src/build/test/examples` and calling `ctest`. This will produce a report on
-which tests passed and which failed for your build. Note that the examples have
-additional dependencies, which may need installing into your virtual
-environment.
+Having built the unit and integration tests, ensure the Python virtual
+environment is active and run them by going to `src/build/test/unit` or the
+corresponding subdirectory of `src/build/test/examples` and calling `ctest`.
+This will produce a report on which tests passed and which failed for your
+build. Note that the examples have additional dependencies, which may need
+installing into your virtual environment.
 
 Alternatively, run the helper script in the root FTorch directory:
 ```
-./run_integration_tests.sh
+./run_test_suite.sh
 ```
 This will automatically install any additional dependencies for the examples.

src/test/unit/CMakeLists.txt

@@ -0,0 +1,12 @@
+cmake_minimum_required(VERSION 3.12)
+cmake_policy (SET CMP0076 NEW)
+
+project("FTorch unit tests" VERSION 1.0.0 LANGUAGES Fortran)
+
+find_package(FTorch)
+message(STATUS "Building with Fortran PyTorch coupling")
+
+find_package(PFUNIT REQUIRED)
+
+add_pfunit_ctest(test_constructors
+  TEST_SOURCES test_constructors.pf LINK_LIBRARIES FTorch::ftorch)

src/test/unit/README.md

@@ -0,0 +1,17 @@
+# Testing
+
+This subdirectory provides automated tests for the FTorch library.
+
+## Pre-requisites for running tests
+
+* FTorch
+* [pFUnit](https://github.com/Goddard-Fortran-Ecosystem/pFUnit) (it is not necessary to build this with MPI support at the moment for these tests).
+
+## Building and running tests
+
+From inside the `tests/` directory
+
+    mkdir build
+    cd build
+    cmake ..
+    make

src/test/unit/test_constructors.pf

@@ -0,0 +1,128 @@
+@test
+subroutine test_torch_tensor_zeros()
+  use pFUnit
+  use ftorch, only: torch_kFloat32, torch_kCPU, torch_tensor, torch_tensor_delete, torch_tensor_to_array, torch_tensor_zeros
+  use ftorch_test_utils, only: assert_allclose
+  use, intrinsic :: iso_fortran_env, only: real32
+  use iso_c_binding, only: c_bool, c_int, c_int64_t, c_null_ptr
+
+  implicit none
+
+  type(torch_tensor) :: tensor
+  integer(c_int) :: ndims
+  integer(c_int64_t), dimension(2) :: tensor_shape
+  integer(c_int) :: dtype
+  integer(c_int) :: device_type
+  integer(c_int) :: device_index
+  logical(c_bool) :: requires_grad
+  real(kind=real32), dimension(:,:), pointer :: out_data
+  real(kind=real32), dimension(2,3) :: expected
+  logical :: test_pass
+
+  ndims = 2
+  tensor_shape = [2, 3]
+  dtype = torch_kFloat32
+  device_type = torch_kCPU
+  device_index = -1
+  requires_grad = .false.
+
+  ! Create a tensor of zeros
+  call torch_tensor_zeros(tensor, ndims, tensor_shape, dtype, device_type, &
+                          device_index, requires_grad)
+
+  ! Check if tensor is not null
+  ! @assertTrue(tensor%p /= c_null_ptr) ! FIXME: compiler not happy with this
+
+  call torch_tensor_to_array(tensor, out_data, shape(expected))
+
+  ! Check that the tensor values are all zero
+  expected(:,:) = 0.0
+  test_pass = assert_allclose(out_data, expected, test_name="test_torch_tensor_zeros")
+  @assertTrue(test_pass)
+
+  ! Cleanup
+  nullify(out_data)
+  call torch_tensor_delete(tensor)
+
+end subroutine test_torch_tensor_zeros
+
+@test
+subroutine test_torch_tensor_ones()
+  use pFUnit
+  use ftorch, only: torch_kFloat32, torch_kCPU, torch_tensor, torch_tensor_delete, torch_tensor_ones, torch_tensor_to_array
+  use ftorch_test_utils, only: assert_allclose
+  use, intrinsic :: iso_fortran_env, only: real32
+  use iso_c_binding, only: c_bool, c_int, c_int64_t, c_null_ptr
+
+  implicit none
+
+  type(torch_tensor) :: tensor
+  integer(c_int) :: ndims
+  integer(c_int64_t), dimension(2) :: tensor_shape
+  integer(c_int) :: dtype
+  integer(c_int) :: device_type
+  integer(c_int) :: device_index
+  logical(c_bool) :: requires_grad
+  real(kind=real32), dimension(:,:), pointer :: out_data
+  real(kind=real32), dimension(2,3) :: expected
+  logical :: test_pass
+
+  ndims = 2
+  tensor_shape = [2, 3]
+  dtype = torch_kFloat32
+  device_type = torch_kCPU
+  device_index = -1
+  requires_grad = .false.
+
+  ! Create tensor of ones
+  call torch_tensor_ones(tensor, ndims, tensor_shape, dtype, device_type, &
+                         device_index, requires_grad)
+
+  ! Check if tensor is not null
+  ! @assertTrue(tensor%p /= c_null_ptr) ! FIXME: compiler not happy with this
+
+  call torch_tensor_to_array(tensor, out_data, shape(expected))
+
+  ! Check that the tensor values are all one
+  expected(:,:) = 1.0
+  test_pass = assert_allclose(out_data, expected, test_name="test_torch_tensor_ones")
+  @assertTrue(test_pass)
+
+  ! Cleanup
+  nullify(out_data)
+  call torch_tensor_delete(tensor)
+
+end subroutine test_torch_tensor_ones
+
+@test
+subroutine test_torch_from_blob()
+  use pFUnit
+  use ftorch
+  use, intrinsic :: iso_c_binding, only : c_ptr, c_int, c_loc
+  use, intrinsic :: iso_c_binding, only: c_int, c_int8_t, c_int16_t, c_int32_t, c_int64_t, c_int64_t, &
+                                        c_float, c_double, c_char, c_ptr, c_null_ptr
+
+  implicit none
+
+  ! generate test data for torch_tensor_from_blob
+  integer(c_int), parameter :: ndims = 2
+  integer(c_int64_t), parameter :: tensor_shape(ndims)  = [2,3] ! 2 rows, 3 columns
+  integer(kind = 4), target :: data1(2,3)  = reshape([1,2,3,4,5,6], [2,3])  ! 2 rows, 3 columns
+  integer(c_int), parameter :: layout(ndims) = [1, 1]
+  integer(c_int) :: dtype  = torch_kInt32
+  integer(c_int) :: device = torch_kCPU
+
+  type(torch_tensor) :: tensor
+  integer(kind = 4) :: data2(3,2) = reshape([1,2,3,4,5,6], [3,2]) ! 3 columns, 2 rows (after reshape)
+
+  ! Smoke test
+  call torch_tensor_from_blob(tensor, c_loc(data1), ndims, tensor_shape, layout, dtype, device)
+
+  ! TODO: would be good to try to compare the data in the tensor to the data in data1
+
+  !@assertEqual(tensor, data2)
+
+  ! Cleanup
+  call torch_tensor_delete(tensor)
+
+end subroutine test_torch_from_blob