From da03b823139d316edbb5a32d31407f8982bbdcbb Mon Sep 17 00:00:00 2001
From: Thinh Nguyen <thinh@vathes.com>
Date: Wed, 16 Aug 2023 18:33:03 -0500
Subject: [PATCH 01/64] support loading multiple caiman results for multi-plane

---
 element_interface/caiman_loader.py | 128 ++++++++++++++++++++++++++++-
 1 file changed, 124 insertions(+), 4 deletions(-)

diff --git a/element_interface/caiman_loader.py b/element_interface/caiman_loader.py
index 3726afd..3ae75c0 100644
--- a/element_interface/caiman_loader.py
+++ b/element_interface/caiman_loader.py
@@ -1,7 +1,7 @@
 import os
 import pathlib
 from datetime import datetime
-
+import re
 import caiman as cm
 import h5py
 import numpy as np
@@ -56,6 +56,119 @@ class CaImAn:
         segmentation_channel: hard-coded to 0
     """
 
+    def __init__(self, caiman_dir: str):
+        """Initialize CaImAn loader class
+
+        Args:
+            caiman_dir (str): string, absolute file path to CaIman directory
+
+        Raises:
+            FileNotFoundError: No CaImAn analysis output file found
+            FileNotFoundError: No CaImAn analysis output found, missing required fields
+        """
+        # ---- Search and verify CaImAn output file exists ----
+        caiman_dir = pathlib.Path(caiman_dir)
+        if not caiman_dir.exists():
+            raise FileNotFoundError("CaImAn directory not found: {}".format(caiman_dir))
+
+        caiman_subdirs = []
+        for fp in caiman_dir.glob("*.hdf5"):
+            with h5py.File(fp, "r") as h5f:
+                if all(s in h5f for s in _required_hdf5_fields):
+                    caiman_subdirs.append(fp.parent)
+
+        if not caiman_subdirs:
+            raise FileNotFoundError(
+                "No CaImAn analysis output file found at {}"
+                " containg all required fields ({})".format(
+                    caiman_dir, _required_hdf5_fields
+                )
+            )
+
+        self.planes = {}
+        for idx, caiman_subdir in enumerate(sorted(caiman_subdirs)):
+            pln_cm = _CaImAn(caiman_subdir.as_posix())
+            pln_idx_match = re.search(r"pln(\d+)_.*")
+            pln_idx = pln_idx_match.groups()[0] if pln_idx_match else idx
+            pln_cm.plane_idx = pln_idx
+            self.planes[pln_idx] = pln_cm
+
+        self._motion_correction = None
+        self._masks = None
+
+        self.creation_time = min(
+            [p.creation_time for p in self.planes.values()]
+        )  # ealiest file creation time
+        self.curation_time = max(
+            [p.curation_time for p in self.planes.values()]
+        )  # most recent curation time
+
+    @property
+    def motion_correction(self):
+        if self._motion_correction is None:
+            self._motion_correction = self.h5f["motion_correction"]
+        return self._motion_correction
+
+    @property
+    def masks(self):
+        if self._masks is None:
+            all_masks = []
+            for pln_idx, _caiman in sorted(self.planes.items()):
+                mask_count = len(all_masks)  # increment mask id from all "plane"
+                all_masks.extend([{**m, "mask_id": m["mask_id"] + mask_count} for m in _caiman.masks])
+
+            self._masks = all_masks
+        return self._masks
+
+    @property
+    def alignment_channel(self):
+        return 0  # hard-code to channel index 0
+
+    @property
+    def segmentation_channel(self):
+        return 0  # hard-code to channel index 0
+
+
+class _CaImAn:
+    """Parse the CaImAn output file
+
+    [CaImAn results doc](https://caiman.readthedocs.io/en/master/Getting_Started.html#result-variables-for-2p-batch-analysis)
+
+    Expecting the following objects:
+        - dims:
+        - dview:
+        - estimates:              Segmentations and traces
+        - mmap_file:
+        - params:                 Input parameters
+        - remove_very_bad_comps:
+        - skip_refinement:
+        - motion_correction:      Motion correction shifts and summary images
+
+    Example:
+        > output_dir = '<imaging_root_data_dir>/subject1/session0/caiman'
+
+        > loaded_dataset = caiman_loader.CaImAn(output_dir)
+
+    Attributes:
+        alignment_channel: hard-coded to 0
+        caiman_fp: file path with all required files:
+            "/motion_correction/reference_image",
+            "/motion_correction/correlation_image",
+            "/motion_correction/average_image",
+            "/motion_correction/max_image",
+            "/estimates/A",
+        cnmf: loaded caiman object; cm.source_extraction.cnmf.cnmf.load_CNMF(caiman_fp)
+        creation_time: file creation time
+        curation_time: file creation time
+        extract_masks: function to extract masks
+        h5f: caiman_fp read as h5py file
+        masks: dict result of extract_masks
+        motion_correction: h5f "motion_correction" property
+        params: cnmf.params
+        segmentation_channel: hard-coded to 0
+        plane_idx: N/A if `is3D` else hard-coded to 0
+    """
+
     def __init__(self, caiman_dir: str):
         """Initialize CaImAn loader class
 
@@ -89,13 +202,20 @@ def __init__(self, caiman_dir: str):
         self.params = self.cnmf.params
 
         self.h5f = h5py.File(self.caiman_fp, "r")
-        self.motion_correction = self.h5f["motion_correction"]
+        self.plane_idx = None if self.params.motion["is3D"] else 0
+        self._motion_correction = None
         self._masks = None
 
         # ---- Metainfo ----
         self.creation_time = datetime.fromtimestamp(os.stat(self.caiman_fp).st_ctime)
         self.curation_time = datetime.fromtimestamp(os.stat(self.caiman_fp).st_ctime)
 
+    @property
+    def motion_correction(self):
+        if self._motion_correction is None:
+            self._motion_correction = self.h5f["motion_correction"]
+        return self._motion_correction
+
     @property
     def masks(self):
         if self._masks is None:
@@ -139,7 +259,7 @@ def extract_masks(self) -> dict:
             else:
                 xpix, ypix = np.unravel_index(ind, self.cnmf.dims, order="F")
                 center_x, center_y = comp_contour["CoM"].astype(int)
-                center_z = 0
+                center_z = self.plane_idx
                 zpix = np.full(len(weights), center_z)
 
             masks.append(
@@ -161,7 +281,7 @@ def extract_masks(self) -> dict:
         return masks
 
 
-def _process_scanimage_tiff(scan_filenames, output_dir="./"):
+def _process_scanimage_tiff(scan_filenames, output_dir="./", split_depths=False):
     """
     Read ScanImage TIFF - reshape into volumetric data based on scanning depths/channels
     Save new TIFF files for each channel - with shape (frame x height x width x depth)

From 9bf69ff8f33af327e8e10450a1d32071b993183e Mon Sep 17 00:00:00 2001
From: Thinh Nguyen <thinh@vathes.com>
Date: Sat, 19 Aug 2023 21:11:52 -0500
Subject: [PATCH 02/64] new caiman loader - handles multi-plane results

---
 element_interface/caiman_loader.py | 210 ++++++++++++++++++++++++++++-
 1 file changed, 204 insertions(+), 6 deletions(-)

diff --git a/element_interface/caiman_loader.py b/element_interface/caiman_loader.py
index 3ae75c0..0b90f38 100644
--- a/element_interface/caiman_loader.py
+++ b/element_interface/caiman_loader.py
@@ -93,9 +93,6 @@ def __init__(self, caiman_dir: str):
             pln_cm.plane_idx = pln_idx
             self.planes[pln_idx] = pln_cm
 
-        self._motion_correction = None
-        self._masks = None
-
         self.creation_time = min(
             [p.creation_time for p in self.planes.values()]
         )  # ealiest file creation time
@@ -103,19 +100,179 @@ def __init__(self, caiman_dir: str):
             [p.curation_time for p in self.planes.values()]
         )  # most recent curation time
 
+        # is this 3D CaImAn analyis or multiple 2D per-plane analysis
+        if len(self.planes) > 1:
+            # if more than one set of caiman result, likely to be multiple 2D per-plane
+            # assert that the "is3D" value are all False for each of the caiman result
+            assert all(p.params.motion["is3D"] is False for p in self.planes.values())
+            self.is3D = False
+            self.is_multiplane = True
+        else:
+            self.is3D = list(self.planes.values())[0].params.motion["is3D"]
+            self.is_multiplane = False
+
+        self._motion_correction = None
+        self._masks = None
+        self._ref_image = None
+        self._mean_image = None
+        self._max_proj_image = None
+        self._correlation_map = None
+
     @property
     def motion_correction(self):
         if self._motion_correction is None:
-            self._motion_correction = self.h5f["motion_correction"]
+            pass
         return self._motion_correction
 
+    def extract_rigid_mc(self):
+        # -- rigid motion correction --
+        rigid_correction = {}
+        for pln_idx, (plane, pln_cm) in enumerate(self.planes.items()):
+            if pln_idx == 0:
+                rigid_correction = {
+                    "x_shifts": pln_cm.motion_correction["shifts_rig"][:, 0],
+                    "y_shifts": pln_cm.motion_correction["shifts_rig"][:, 1],
+                }
+                rigid_correction["x_std"] = np.nanstd(
+                    rigid_correction["x_shifts"].flatten()
+                )
+                rigid_correction["y_std"] = np.nanstd(
+                    rigid_correction["y_shifts"].flatten()
+                )
+            else:
+                rigid_correction["x_shifts"] = np.vstack(
+                    [
+                        rigid_correction["x_shifts"],
+                        pln_cm.motion_correction["shifts_rig"][:, 0],
+                    ]
+                )
+                rigid_correction["x_std"] = np.nanstd(
+                    rigid_correction["x_shifts"].flatten()
+                )
+                rigid_correction["y_shifts"] = np.vstack(
+                    [
+                        rigid_correction["y_shifts"],
+                        pln_cm.motion_correction["shifts_rig"][:, 1],
+                    ]
+                )
+                rigid_correction["y_std"] = np.nanstd(
+                    rigid_correction["y_shifts"].flatten()
+                )
+
+        if not self.is_multiplane:
+            pln_cm = list(self.planes.values())[0]
+            rigid_correction["z_shifts"] = (
+                pln_cm.motion_correction["shifts_rig"][:, 2]
+                if self.is3D
+                else np.full_like(rigid_correction["x_shifts"], 0)
+            )
+            rigid_correction["z_std"] = (
+                np.nanstd(pln_cm.motion_correction["shifts_rig"][:, 2])
+                if self.is3D
+                else np.nan
+            )
+        else:
+            rigid_correction["z_shifts"] = np.full_like(rigid_correction["x_shifts"], 0)
+            rigid_correction["z_std"] = np.nan
+
+        rigid_correction["outlier_frames"] = None
+
+        return rigid_correction
+
+    def extract_pw_rigid_mc(self):
+        # -- piece-wise rigid motion correction --
+        nonrigid_correction, nonrigid_blocks = {}
+        for pln_idx, (plane, pln_cm) in enumerate(self.planes.items()):
+            if pln_idx == 0:
+                nonrigid_correction = {
+                    "block_height": (
+                        pln_cm.params.motion["strides"][0]
+                        + pln_cm.params.motion["overlaps"][0]
+                    ),
+                    "block_width": (
+                        pln_cm.params.motion["strides"][1]
+                        + pln_cm.params.motion["overlaps"][1]
+                    ),
+                    "block_depth": 1,
+                    "block_count_x": len(
+                        set(pln_cm.motion_correction["coord_shifts_els"][:, 0])
+                    ),
+                    "block_count_y": len(
+                        set(pln_cm.motion_correction["coord_shifts_els"][:, 2])
+                    ),
+                    "block_count_z": len(self.planes),
+                    "outlier_frames": None,
+                }
+            for b_id in range(len(pln_cm.motion_correction["x_shifts_els"][0, :])):
+                if b_id in nonrigid_blocks:
+                    nonrigid_blocks[b_id]["x_shifts"] = np.vstack(
+                        [
+                            nonrigid_blocks[b_id]["x_shifts"],
+                            pln_cm.motion_correction["x_shifts_els"][:, b_id],
+                        ]
+                    )
+                    nonrigid_blocks[b_id]["x_std"] = np.nanstd(
+                        nonrigid_blocks[b_id]["x_shifts"].flatten()
+                    )
+                    nonrigid_blocks[b_id]["y_shifts"] = np.vstack(
+                        [
+                            nonrigid_blocks[b_id]["y_shifts"],
+                            pln_cm.motion_correction["y_shifts_els"][:, b_id],
+                        ]
+                    )
+                    nonrigid_blocks[b_id]["y_std"] = np.nanstd(
+                        nonrigid_blocks[b_id]["y_shifts"].flatten()
+                    )
+                    nonrigid_blocks[b_id]["z_shifts"] = np.vstack(
+                        [
+                            nonrigid_blocks[b_id]["z_shifts"],
+                            np.full_like(
+                                pln_cm.motion_correction["x_shifts_els"][:, b_id],
+                                0,
+                            ),
+                        ]
+                    )
+                else:
+                    nonrigid_blocks[b_id] = {
+                        "block_id": b_id,
+                        "block_x": np.arange(
+                            *pln_cm.motion_correction["coord_shifts_els"][b_id, 0:2]
+                        ),
+                        "block_y": np.arange(
+                            *pln_cm.motion_correction["coord_shifts_els"][b_id, 2:4]
+                        ),
+                        "block_z": np.full_like(
+                            np.arange(
+                                *pln_cm.motion_correction["coord_shifts_els"][b_id, 0:2]
+                            ),
+                            pln_idx,
+                        ),
+                        "x_shifts": pln_cm.motion_correction["x_shifts_els"][:, b_id],
+                        "y_shifts": pln_cm.motion_correction["y_shifts_els"][:, b_id],
+                        "z_shifts": np.full_like(
+                            pln_cm.motion_correction["x_shifts_els"][:, b_id],
+                            0,
+                        ),
+                        "x_std": np.nanstd(
+                            pln_cm.motion_correction["x_shifts_els"][:, b_id]
+                        ),
+                        "y_std": np.nanstd(
+                            pln_cm.motion_correction["y_shifts_els"][:, b_id]
+                        ),
+                        "z_std": np.nan,
+                    }
+
+        return nonrigid_correction, nonrigid_blocks
+
     @property
     def masks(self):
         if self._masks is None:
             all_masks = []
-            for pln_idx, _caiman in sorted(self.planes.items()):
+            for pln_idx, pln_cm in sorted(self.planes.items()):
                 mask_count = len(all_masks)  # increment mask id from all "plane"
-                all_masks.extend([{**m, "mask_id": m["mask_id"] + mask_count} for m in _caiman.masks])
+                all_masks.extend(
+                    [{**m, "mask_id": m["mask_id"] + mask_count} for m in pln_cm.masks]
+                )
 
             self._masks = all_masks
         return self._masks
@@ -128,6 +285,47 @@ def alignment_channel(self):
     def segmentation_channel(self):
         return 0  # hard-code to channel index 0
 
+    # -- image property --
+
+    def _get_image(self, img_type):
+        if not self.is_multiplane:
+            pln_cm = list(self.planes.values())[0]
+            _img = (
+                pln_cm.motion_correction[img_type].transpose()
+                if self.is3D
+                else pln_cm.motion_correction[img_type][...][np.newaxis, ...]
+            )
+        else:
+            _img = np.dstack(
+                pln_cm.motion_correction[img_type][...]
+                for pln_cm in self.planes.values()
+            )
+        return _img
+
+    @property
+    def ref_image(self):
+        if self._ref_image is None:
+            self._ref_image = self._get_image("reference_image")
+        return self._ref_image
+
+    @property
+    def mean_image(self):
+        if self._mean_image is None:
+            self._mean_image = self._get_image("average_image")
+        return self._mean_image
+
+    @property
+    def max_proj_image(self):
+        if self._max_proj_image is None:
+            self._max_proj_image = self._get_image("max_image")
+        return self._max_proj_image
+
+    @property
+    def correlation_map(self):
+        if self._correlation_map is None:
+            self._correlation_map = self._get_image("correlation_image")
+        return self._correlation_map
+
 
 class _CaImAn:
     """Parse the CaImAn output file

From 05d7c3544665a9345b1732abc43d0311fadff143 Mon Sep 17 00:00:00 2001
From: Thinh Nguyen <thinh@vathes.com>
Date: Sat, 19 Aug 2023 21:18:34 -0500
Subject: [PATCH 03/64] improve non-rigid motion correction loading

---
 element_interface/caiman_loader.py | 103 +++++++++++++++--------------
 1 file changed, 52 insertions(+), 51 deletions(-)

diff --git a/element_interface/caiman_loader.py b/element_interface/caiman_loader.py
index 0b90f38..2e73c3d 100644
--- a/element_interface/caiman_loader.py
+++ b/element_interface/caiman_loader.py
@@ -111,6 +111,11 @@ def __init__(self, caiman_dir: str):
             self.is3D = list(self.planes.values())[0].params.motion["is3D"]
             self.is_multiplane = False
 
+        if self.is_multiplane and self.is3D:
+            raise NotImplementedError(
+                f"Unable to load CaImAn results mixed between 3D and multi-plane analysis"
+            )
+
         self._motion_correction = None
         self._masks = None
         self._ref_image = None
@@ -183,6 +188,7 @@ def extract_pw_rigid_mc(self):
         # -- piece-wise rigid motion correction --
         nonrigid_correction, nonrigid_blocks = {}
         for pln_idx, (plane, pln_cm) in enumerate(self.planes.items()):
+            block_count = len(nonrigid_blocks)
             if pln_idx == 0:
                 nonrigid_correction = {
                     "block_height": (
@@ -204,63 +210,58 @@ def extract_pw_rigid_mc(self):
                     "outlier_frames": None,
                 }
             for b_id in range(len(pln_cm.motion_correction["x_shifts_els"][0, :])):
-                if b_id in nonrigid_blocks:
-                    nonrigid_blocks[b_id]["x_shifts"] = np.vstack(
-                        [
-                            nonrigid_blocks[b_id]["x_shifts"],
-                            pln_cm.motion_correction["x_shifts_els"][:, b_id],
-                        ]
-                    )
-                    nonrigid_blocks[b_id]["x_std"] = np.nanstd(
-                        nonrigid_blocks[b_id]["x_shifts"].flatten()
-                    )
-                    nonrigid_blocks[b_id]["y_shifts"] = np.vstack(
-                        [
-                            nonrigid_blocks[b_id]["y_shifts"],
-                            pln_cm.motion_correction["y_shifts_els"][:, b_id],
-                        ]
-                    )
-                    nonrigid_blocks[b_id]["y_std"] = np.nanstd(
-                        nonrigid_blocks[b_id]["y_shifts"].flatten()
-                    )
-                    nonrigid_blocks[b_id]["z_shifts"] = np.vstack(
-                        [
-                            nonrigid_blocks[b_id]["z_shifts"],
-                            np.full_like(
-                                pln_cm.motion_correction["x_shifts_els"][:, b_id],
-                                0,
-                            ),
-                        ]
-                    )
-                else:
-                    nonrigid_blocks[b_id] = {
-                        "block_id": b_id,
-                        "block_x": np.arange(
-                            *pln_cm.motion_correction["coord_shifts_els"][b_id, 0:2]
-                        ),
-                        "block_y": np.arange(
-                            *pln_cm.motion_correction["coord_shifts_els"][b_id, 2:4]
-                        ),
-                        "block_z": np.full_like(
+                b_id += block_count
+                nonrigid_blocks[b_id] = {
+                    "block_id": b_id,
+                    "block_x": np.arange(
+                        *pln_cm.motion_correction["coord_shifts_els"][b_id, 0:2]
+                    ),
+                    "block_y": np.arange(
+                        *pln_cm.motion_correction["coord_shifts_els"][b_id, 2:4]
+                    ),
+                    "block_z": (
+                        np.arange(
+                            *pln_cm.motion_correction["coord_shifts_els"][b_id, 4:6]
+                        )
+                        if self.is3D
+                        else np.full_like(
                             np.arange(
                                 *pln_cm.motion_correction["coord_shifts_els"][b_id, 0:2]
                             ),
                             pln_idx,
-                        ),
-                        "x_shifts": pln_cm.motion_correction["x_shifts_els"][:, b_id],
-                        "y_shifts": pln_cm.motion_correction["y_shifts_els"][:, b_id],
-                        "z_shifts": np.full_like(
+                        )
+                    ),
+                    "x_shifts": pln_cm.motion_correction["x_shifts_els"][:, b_id],
+                    "y_shifts": pln_cm.motion_correction["y_shifts_els"][:, b_id],
+                    "z_shifts": (
+                        pln_cm.motion_correction["z_shifts_els"][:, b_id]
+                        if self.is3D
+                        else np.full_like(
                             pln_cm.motion_correction["x_shifts_els"][:, b_id],
                             0,
-                        ),
-                        "x_std": np.nanstd(
-                            pln_cm.motion_correction["x_shifts_els"][:, b_id]
-                        ),
-                        "y_std": np.nanstd(
-                            pln_cm.motion_correction["y_shifts_els"][:, b_id]
-                        ),
-                        "z_std": np.nan,
-                    }
+                        )
+                    ),
+                    "x_std": np.nanstd(
+                        pln_cm.motion_correction["x_shifts_els"][:, b_id]
+                    ),
+                    "y_std": np.nanstd(
+                        pln_cm.motion_correction["y_shifts_els"][:, b_id]
+                    ),
+                    "z_std": (
+                        np.nanstd(pln_cm.motion_correction["z_shifts_els"][:, b_id])
+                        if self.is3D
+                        else np.nan
+                    ),
+                }
+
+        if not self.is_multiplane and self.is3D:
+            pln_cm = list(self.planes.values())[0]
+            nonrigid_correction["block_depth"] = (
+                pln_cm.params.motion["strides"][2] + pln_cm.params.motion["overlaps"][2]
+            )
+            nonrigid_correction["block_count_z"] = len(
+                set(pln_cm.motion_correction["coord_shifts_els"][:, 4])
+            )
 
         return nonrigid_correction, nonrigid_blocks
 

From b050265753080f2c7632b7b8a0a0734bbd239b19 Mon Sep 17 00:00:00 2001
From: Thinh Nguyen <thinh@vathes.com>
Date: Tue, 22 Aug 2023 09:22:27 -0500
Subject: [PATCH 04/64] added routine to combine single-page tiffs into one
 bigtiff

---
 element_interface/caiman_loader.py       |  6 +-
 element_interface/prairie_view_loader.py | 74 +++++++++++++++++++-----
 2 files changed, 64 insertions(+), 16 deletions(-)

diff --git a/element_interface/caiman_loader.py b/element_interface/caiman_loader.py
index 2e73c3d..efdfc05 100644
--- a/element_interface/caiman_loader.py
+++ b/element_interface/caiman_loader.py
@@ -291,17 +291,17 @@ def segmentation_channel(self):
     def _get_image(self, img_type):
         if not self.is_multiplane:
             pln_cm = list(self.planes.values())[0]
-            _img = (
+            img_ = (
                 pln_cm.motion_correction[img_type].transpose()
                 if self.is3D
                 else pln_cm.motion_correction[img_type][...][np.newaxis, ...]
             )
         else:
-            _img = np.dstack(
+            img_ = np.dstack(
                 pln_cm.motion_correction[img_type][...]
                 for pln_cm in self.planes.values()
             )
-        return _img
+        return img_
 
     @property
     def ref_image(self):
diff --git a/element_interface/prairie_view_loader.py b/element_interface/prairie_view_loader.py
index ee306cd..746829b 100644
--- a/element_interface/prairie_view_loader.py
+++ b/element_interface/prairie_view_loader.py
@@ -1,12 +1,13 @@
+import os
 import pathlib
 from pathlib import Path
 import xml.etree.ElementTree as ET
 from datetime import datetime
 import numpy as np
+import tifffile
 
 
 class PrairieViewMeta:
-
     def __init__(self, prairieview_dir: str):
         """Initialize PrairieViewMeta loader class
 
@@ -37,25 +38,72 @@ def meta(self):
             self._meta = _extract_prairieview_metadata(self.xml_file)
         return self._meta
 
-    def get_prairieview_files(self, plane_idx=None, channel=None):
+    def get_prairieview_filenames(
+        self, plane_idx=None, channel=None, return_pln_chn=False
+    ):
+        """
+        Extract from metadata the set of tiff files specific to the specified "plane_idx" and "channel"
+        Args:
+            plane_idx: int - plane index
+            channel: int - channel
+            return_pln_chn: bool - if True, returns (filenames, plane_idx, channel), else returns `filenames`
+
+        Returns: List[str] - the set of tiff files specific to the specified "plane_idx" and "channel"
+        """
         if plane_idx is None:
-            if self.meta['num_planes'] > 1:
-                raise ValueError(f"Please specify 'plane_idx' - Plane indices: {self.meta['plane_indices']}")
+            if self.meta["num_planes"] > 1:
+                raise ValueError(
+                    f"Please specify 'plane_idx' - Plane indices: {self.meta['plane_indices']}"
+                )
             else:
-                plane_idx = self.meta['plane_indices'][0]
+                plane_idx = self.meta["plane_indices"][0]
         else:
-            assert plane_idx in self.meta['plane_indices'], f"Invalid 'plane_idx' - Plane indices: {self.meta['plane_indices']}"
+            assert (
+                plane_idx in self.meta["plane_indices"]
+            ), f"Invalid 'plane_idx' - Plane indices: {self.meta['plane_indices']}"
 
         if channel is None:
-            if self.meta['num_channels'] > 1:
-                raise ValueError(f"Please specify 'channel' - Channels: {self.meta['channels']}")
+            if self.meta["num_channels"] > 1:
+                raise ValueError(
+                    f"Please specify 'channel' - Channels: {self.meta['channels']}"
+                )
             else:
-                plane_idx = self.meta['channels'][0]
+                plane_idx = self.meta["channels"][0]
         else:
-            assert channel in self.meta['channels'], f"Invalid 'channel' - Channels: {self.meta['channels']}"
+            assert (
+                channel in self.meta["channels"]
+            ), f"Invalid 'channel' - Channels: {self.meta['channels']}"
+
+        frames = self._xml_root.findall(
+            f".//Sequence/Frame/[@index='{plane_idx}']/File/[@channel='{channel}']"
+        )
 
-        frames = self._xml_root.findall(f".//Sequence/Frame/[@index='{plane_idx}']/File/[@channel='{channel}']")
-        return [f.attrib['filename'] for f in frames]
+        fnames = [f.attrib["filename"] for f in frames]
+        return fnames if not return_pln_chn else (fnames, plane_idx, channel)
+
+    def write_single_tiff(
+        self, plane_idx=None, channel=None, output_prefix=None, output_dir="./"
+    ):
+        tiff_names, plane_idx, channel = self.get_prairieview_filenames(
+            plane_idx=plane_idx, channel=channel, return_pln_chn=True
+        )
+        combined_data = []
+        for input_file in tiff_names:
+            with tifffile.TiffFile(self.prairieview_dir / input_file) as tffl:
+                assert len(tffl.pages) == 1
+                combined_data.append(tffl.asarray())
+        combined_data = np.dstack(combined_data).transpose(
+            2, 0, 1
+        )  # (frame x height x width)
+
+        if output_prefix is None:
+            output_prefix = os.path.commonprefix(tiff_names)
+
+        tifffile.imwrite(
+            Path(output_dir) / f"{output_prefix}_pln{plane_idx}_chn{channel}",
+            combined_data,
+            metadata={"axes": "TXY", "'fps'": self.meta["frame_rate"]},
+        )
 
 
 def _extract_prairieview_metadata(xml_filepath: str):
@@ -159,7 +207,7 @@ def _extract_prairieview_metadata(xml_filepath: str):
             ".//Sequence/[@cycle='1']/Frame/[@index='1']/PVStateShard/PVStateValue/[@key='positionCurrent']/SubindexedValues/[@index='ZAxis']/SubindexedValue"
         )
 
-        # If more than one Z-axis controllers are found, 
+        # If more than one Z-axis controllers are found,
         # check which controller is changing z_field depth. Only 1 controller
         # must change depths.
         if len(z_controllers) > 1:

From 512e5afee28be7ddb9d0baf9b86480a916f62b40 Mon Sep 17 00:00:00 2001
From: Thinh Nguyen <thinh@vathes.com>
Date: Tue, 22 Aug 2023 14:42:29 -0500
Subject: [PATCH 05/64] run_caimain - more robust

---
 element_interface/run_caiman.py | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/element_interface/run_caiman.py b/element_interface/run_caiman.py
index eb480a9..6cb1559 100644
--- a/element_interface/run_caiman.py
+++ b/element_interface/run_caiman.py
@@ -41,12 +41,19 @@ def run_caiman(
         backend="local", n_processes=None, single_thread=False
     )
 
-    cnm = CNMF(n_processes, params=opts, dview=dview)
-    cnmf_output, mc_output = cnm.fit_file(
-        motion_correct=True, include_eval=True, output_dir=output_dir, return_mc=True
-    )
-
-    cm.stop_server(dview=dview)
+    try:
+        cnm = CNMF(n_processes, params=opts, dview=dview)
+        cnmf_output, mc_output = cnm.fit_file(
+            motion_correct=True,
+            include_eval=True,
+            output_dir=output_dir,
+            return_mc=True,
+        )
+    except Exception as e:
+        dview.terminate()
+        raise e
+    else:
+        cm.stop_server(dview=dview)
 
     cnmf_output_file = pathlib.Path(cnmf_output.mmap_file[:-4] + "hdf5")
     assert cnmf_output_file.exists()

From 7d6ea0e4bc8af336394c47e9703a8dc0d71550d9 Mon Sep 17 00:00:00 2001
From: Thinh Nguyen <thinh@vathes.com>
Date: Tue, 22 Aug 2023 14:42:40 -0500
Subject: [PATCH 06/64] add `caiman_compatible` mode

---
 element_interface/prairie_view_loader.py | 60 +++++++++++++++++-------
 1 file changed, 44 insertions(+), 16 deletions(-)

diff --git a/element_interface/prairie_view_loader.py b/element_interface/prairie_view_loader.py
index 746829b..5c4f06e 100644
--- a/element_interface/prairie_view_loader.py
+++ b/element_interface/prairie_view_loader.py
@@ -68,7 +68,7 @@ def get_prairieview_filenames(
                     f"Please specify 'channel' - Channels: {self.meta['channels']}"
                 )
             else:
-                plane_idx = self.meta["channels"][0]
+                channel = self.meta["channels"][0]
         else:
             assert (
                 channel in self.meta["channels"]
@@ -82,28 +82,56 @@ def get_prairieview_filenames(
         return fnames if not return_pln_chn else (fnames, plane_idx, channel)
 
     def write_single_tiff(
-        self, plane_idx=None, channel=None, output_prefix=None, output_dir="./"
+        self,
+        plane_idx=None,
+        channel=None,
+        output_prefix=None,
+        output_dir="./",
+        caiman_compatible=False,  # if True, save the movie as a single page (frame x height x width)
+        overwrite=False,
     ):
         tiff_names, plane_idx, channel = self.get_prairieview_filenames(
             plane_idx=plane_idx, channel=channel, return_pln_chn=True
         )
-        combined_data = []
-        for input_file in tiff_names:
-            with tifffile.TiffFile(self.prairieview_dir / input_file) as tffl:
-                assert len(tffl.pages) == 1
-                combined_data.append(tffl.asarray())
-        combined_data = np.dstack(combined_data).transpose(
-            2, 0, 1
-        )  # (frame x height x width)
-
         if output_prefix is None:
             output_prefix = os.path.commonprefix(tiff_names)
-
-        tifffile.imwrite(
-            Path(output_dir) / f"{output_prefix}_pln{plane_idx}_chn{channel}",
-            combined_data,
-            metadata={"axes": "TXY", "'fps'": self.meta["frame_rate"]},
+        output_tiff_fullpath = (
+            Path(output_dir)
+            / f"{output_prefix}_pln{plane_idx}_chn{channel}{'.ome' if not caiman_compatible else ''}.tif"
         )
+        if output_tiff_fullpath.exists() and not overwrite:
+            return output_tiff_fullpath
+
+        if not caiman_compatible:
+            with tifffile.TiffWriter(
+                output_tiff_fullpath,
+                bigtiff=True,
+            ) as tiff_writer:
+                for input_file in tiff_names:
+                    with tifffile.TiffFile(self.prairieview_dir / input_file) as tffl:
+                        assert len(tffl.pages) == 1
+                        tiff_writer.write(
+                            tffl.pages[0].asarray(),
+                            metadata={"axes": "YX", "'fps'": self.meta["frame_rate"]},
+                        )
+        else:
+            combined_data = []
+            for input_file in tiff_names:
+                with tifffile.TiffFile(self.prairieview_dir / input_file) as tffl:
+                    assert len(tffl.pages) == 1
+                    combined_data.append(tffl.pages[0].asarray())
+            combined_data = np.dstack(combined_data).transpose(
+                2, 0, 1
+            )  # (frame x height x width)
+
+            tifffile.imwrite(
+                output_tiff_fullpath,
+                combined_data,
+                metadata={"axes": "TYX", "'fps'": self.meta["frame_rate"]},
+                bigtiff=True,
+            )
+
+        return output_tiff_fullpath
 
 
 def _extract_prairieview_metadata(xml_filepath: str):

From 55e5a5c3fa50ebcdedc1f660e64f9a9b7d98a09e Mon Sep 17 00:00:00 2001
From: Thinh Nguyen <thinh@vathes.com>
Date: Thu, 24 Aug 2023 12:13:05 -0500
Subject: [PATCH 07/64] minor cleanup, version bump

---
 element_interface/prairie_view_loader.py | 2 +-
 element_interface/version.py             | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/element_interface/prairie_view_loader.py b/element_interface/prairie_view_loader.py
index 5c4f06e..1a05139 100644
--- a/element_interface/prairie_view_loader.py
+++ b/element_interface/prairie_view_loader.py
@@ -81,7 +81,7 @@ def get_prairieview_filenames(
         fnames = [f.attrib["filename"] for f in frames]
         return fnames if not return_pln_chn else (fnames, plane_idx, channel)
 
-    def write_single_tiff(
+    def write_single_bigtiff(
         self,
         plane_idx=None,
         channel=None,
diff --git a/element_interface/version.py b/element_interface/version.py
index 0da8726..70aab85 100644
--- a/element_interface/version.py
+++ b/element_interface/version.py
@@ -1,3 +1,3 @@
 """Package metadata"""
 
-__version__ = "0.6.1"
+__version__ = "0.7.0"

From cb08034edfa1e94f509513010e802b01bb431c31 Mon Sep 17 00:00:00 2001
From: Thinh Nguyen <thinh@vathes.com>
Date: Fri, 25 Aug 2023 09:56:32 -0500
Subject: [PATCH 08/64] Update caiman_loader.py

---
 element_interface/caiman_loader.py | 75 ++++++++++++++----------------
 1 file changed, 35 insertions(+), 40 deletions(-)

diff --git a/element_interface/caiman_loader.py b/element_interface/caiman_loader.py
index efdfc05..580e194 100644
--- a/element_interface/caiman_loader.py
+++ b/element_interface/caiman_loader.py
@@ -18,42 +18,10 @@
 
 
 class CaImAn:
-    """Parse the CaImAn output file
-
-    [CaImAn results doc](https://caiman.readthedocs.io/en/master/Getting_Started.html#result-variables-for-2p-batch-analysis)
-
-    Expecting the following objects:
-        - dims:
-        - dview:
-        - estimates:              Segmentations and traces
-        - mmap_file:
-        - params:                 Input parameters
-        - remove_very_bad_comps:
-        - skip_refinement:
-        - motion_correction:      Motion correction shifts and summary images
-
-    Example:
-        > output_dir = '<imaging_root_data_dir>/subject1/session0/caiman'
-
-        > loaded_dataset = caiman_loader.CaImAn(output_dir)
-
-    Attributes:
-        alignment_channel: hard-coded to 0
-        caiman_fp: file path with all required files:
-            "/motion_correction/reference_image",
-            "/motion_correction/correlation_image",
-            "/motion_correction/average_image",
-            "/motion_correction/max_image",
-            "/estimates/A",
-        cnmf: loaded caiman object; cm.source_extraction.cnmf.cnmf.load_CNMF(caiman_fp)
-        creation_time: file creation time
-        curation_time: file creation time
-        extract_masks: function to extract masks
-        h5f: caiman_fp read as h5py file
-        masks: dict result of extract_masks
-        motion_correction: h5f "motion_correction" property
-        params: cnmf.params
-        segmentation_channel: hard-coded to 0
+    """
+    Loader class for CaImAn analysis results
+    A top level aggregator of multiple set of CaImAn results (e.g. multi-plane analysis)
+    Calling _CaImAn (see below) under the hood
     """
 
     def __init__(self, caiman_dir: str):
@@ -85,13 +53,16 @@ def __init__(self, caiman_dir: str):
                 )
             )
 
-        self.planes = {}
+        # Extract CaImAn results from all planes, sorted by plane index
+        _planes_caiman = {}
         for idx, caiman_subdir in enumerate(sorted(caiman_subdirs)):
             pln_cm = _CaImAn(caiman_subdir.as_posix())
             pln_idx_match = re.search(r"pln(\d+)_.*")
             pln_idx = pln_idx_match.groups()[0] if pln_idx_match else idx
             pln_cm.plane_idx = pln_idx
-            self.planes[pln_idx] = pln_cm
+            _planes_caiman[pln_idx] = pln_cm
+        sorted_pln_ind = sorted(list(_planes_caiman.keys()))
+        self.planes = {k: _planes_caiman[k] for k in sorted_pln_ind}
 
         self.creation_time = min(
             [p.creation_time for p in self.planes.values()]
@@ -123,10 +94,22 @@ def __init__(self, caiman_dir: str):
         self._max_proj_image = None
         self._correlation_map = None
 
+    @property
+    def is_pw_rigid(self):
+        pw_rigid = set(p.params.motion["pw_rigid"] for p in self.planes.values())
+        assert (
+            len(pw_rigid) == 1
+        ), f"Unable to load CaImAn results mixed between rigid and pw_rigid motion correction"
+        return pw_rigid.pop()
+
     @property
     def motion_correction(self):
         if self._motion_correction is None:
-            pass
+            self._motion_correction = (
+                self.extract_pw_rigid_mc()
+                if self.is_pw_rigid
+                else self.extract_rigid_mc()
+            )
         return self._motion_correction
 
     def extract_rigid_mc(self):
@@ -272,7 +255,19 @@ def masks(self):
             for pln_idx, pln_cm in sorted(self.planes.items()):
                 mask_count = len(all_masks)  # increment mask id from all "plane"
                 all_masks.extend(
-                    [{**m, "mask_id": m["mask_id"] + mask_count} for m in pln_cm.masks]
+                    [
+                        {
+                            **m,
+                            "mask_id": m["mask_id"] + mask_count,
+                            "orig_mask_id": m["mask_id"],
+                            "accepted": (
+                                m["mask_id"] in pln_cm.cnmf.estimates.idx_components
+                                if pln_cm.cnmf.estimates.idx_components is not None
+                                else False
+                            ),
+                        }
+                        for m in pln_cm.masks
+                    ]
                 )
 
             self._masks = all_masks

From 85078fdbe7dc71b1885e94ee93a460898094ffd6 Mon Sep 17 00:00:00 2001
From: kushalbakshi <kushalbakshi@gmail.com>
Date: Mon, 28 Aug 2023 11:22:58 -0500
Subject: [PATCH 09/64] Fix regex error

---
 element_interface/caiman_loader.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/element_interface/caiman_loader.py b/element_interface/caiman_loader.py
index 580e194..1743aa7 100644
--- a/element_interface/caiman_loader.py
+++ b/element_interface/caiman_loader.py
@@ -57,7 +57,7 @@ def __init__(self, caiman_dir: str):
         _planes_caiman = {}
         for idx, caiman_subdir in enumerate(sorted(caiman_subdirs)):
             pln_cm = _CaImAn(caiman_subdir.as_posix())
-            pln_idx_match = re.search(r"pln(\d+)_.*")
+            pln_idx_match = re.search(r"pln(\d+)_.*", caiman_subdir)
             pln_idx = pln_idx_match.groups()[0] if pln_idx_match else idx
             pln_cm.plane_idx = pln_idx
             _planes_caiman[pln_idx] = pln_cm

From 7b815ff4f8a8d710c02556aee5581f579956d96e Mon Sep 17 00:00:00 2001
From: Thinh Nguyen <thinh@vathes.com>
Date: Mon, 28 Aug 2023 12:30:46 -0500
Subject: [PATCH 10/64] bugfix

---
 element_interface/caiman_loader.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/element_interface/caiman_loader.py b/element_interface/caiman_loader.py
index 580e194..667df5a 100644
--- a/element_interface/caiman_loader.py
+++ b/element_interface/caiman_loader.py
@@ -57,7 +57,7 @@ def __init__(self, caiman_dir: str):
         _planes_caiman = {}
         for idx, caiman_subdir in enumerate(sorted(caiman_subdirs)):
             pln_cm = _CaImAn(caiman_subdir.as_posix())
-            pln_idx_match = re.search(r"pln(\d+)_.*")
+            pln_idx_match = re.search(r"pln(\d+)_.*", caiman_subdir.stem)
             pln_idx = pln_idx_match.groups()[0] if pln_idx_match else idx
             pln_cm.plane_idx = pln_idx
             _planes_caiman[pln_idx] = pln_cm

From 0bf78ed542bc8a9dba572adcf3b8d54eca985a8f Mon Sep 17 00:00:00 2001
From: kushalbakshi <kushalbakshi@gmail.com>
Date: Mon, 28 Aug 2023 12:58:40 -0500
Subject: [PATCH 11/64] Resolve TypeError: Add `.stem()`

---
 element_interface/caiman_loader.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/element_interface/caiman_loader.py b/element_interface/caiman_loader.py
index 1743aa7..667df5a 100644
--- a/element_interface/caiman_loader.py
+++ b/element_interface/caiman_loader.py
@@ -57,7 +57,7 @@ def __init__(self, caiman_dir: str):
         _planes_caiman = {}
         for idx, caiman_subdir in enumerate(sorted(caiman_subdirs)):
             pln_cm = _CaImAn(caiman_subdir.as_posix())
-            pln_idx_match = re.search(r"pln(\d+)_.*", caiman_subdir)
+            pln_idx_match = re.search(r"pln(\d+)_.*", caiman_subdir.stem)
             pln_idx = pln_idx_match.groups()[0] if pln_idx_match else idx
             pln_cm.plane_idx = pln_idx
             _planes_caiman[pln_idx] = pln_cm

From 706df0484d53dbc2aca1a29622090d1d66879e96 Mon Sep 17 00:00:00 2001
From: kushalbakshi <kushalbakshi@gmail.com>
Date: Mon, 28 Aug 2023 17:20:43 -0500
Subject: [PATCH 12/64] Debug `FileNotFoundError` at output_dir

---
 element_interface/caiman_loader.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/element_interface/caiman_loader.py b/element_interface/caiman_loader.py
index 667df5a..a87d9eb 100644
--- a/element_interface/caiman_loader.py
+++ b/element_interface/caiman_loader.py
@@ -40,7 +40,7 @@ def __init__(self, caiman_dir: str):
             raise FileNotFoundError("CaImAn directory not found: {}".format(caiman_dir))
 
         caiman_subdirs = []
-        for fp in caiman_dir.glob("*.hdf5"):
+        for fp in caiman_dir.rglob("*.hdf5"):
             with h5py.File(fp, "r") as h5f:
                 if all(s in h5f for s in _required_hdf5_fields):
                     caiman_subdirs.append(fp.parent)

From df0d9b8edc6da696359a3da12d1e356ab3ab081b Mon Sep 17 00:00:00 2001
From: kushalbakshi <kushalbakshi@gmail.com>
Date: Tue, 29 Aug 2023 15:00:38 -0500
Subject: [PATCH 13/64] Raise exception to debug summary image dimensions

---
 element_interface/caiman_loader.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/element_interface/caiman_loader.py b/element_interface/caiman_loader.py
index a87d9eb..b7f0b50 100644
--- a/element_interface/caiman_loader.py
+++ b/element_interface/caiman_loader.py
@@ -296,6 +296,7 @@ def _get_image(self, img_type):
                 pln_cm.motion_correction[img_type][...]
                 for pln_cm in self.planes.values()
             )
+            raise Exception("Debug summary images")
         return img_
 
     @property

From 13e6e946eec2d6e4f6ced579a205ea26ebb969e4 Mon Sep 17 00:00:00 2001
From: kushalbakshi <kushalbakshi@gmail.com>
Date: Tue, 29 Aug 2023 15:14:18 -0500
Subject: [PATCH 14/64] Revert previous commit

---
 element_interface/caiman_loader.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/element_interface/caiman_loader.py b/element_interface/caiman_loader.py
index b7f0b50..a87d9eb 100644
--- a/element_interface/caiman_loader.py
+++ b/element_interface/caiman_loader.py
@@ -296,7 +296,6 @@ def _get_image(self, img_type):
                 pln_cm.motion_correction[img_type][...]
                 for pln_cm in self.planes.values()
             )
-            raise Exception("Debug summary images")
         return img_
 
     @property

From 20188ad02eb0c02e74886ebdc54df73f1517b2dd Mon Sep 17 00:00:00 2001
From: kushalbakshi <kushalbakshi@gmail.com>
Date: Tue, 5 Sep 2023 16:01:50 -0500
Subject: [PATCH 15/64] Update CHANGELOG for #88

---
 CHANGELOG.md | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index b7dfd0b..54478f9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,11 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and
 [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) convention.
 
 
+## [0.7.0] - 2023-09-05
+
++ Add - multi-plane `caiman_loader.py` to process multi-plane tiffs
++ Update - DANDI upload utility
+
 ## [0.6.1] - 2023-08-02
 
 + Update DANDI upload funtionality to improve useability
@@ -83,6 +88,7 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and
 
 + Add - Readers for: `ScanImage`, `Suite2p`, `CaImAn`.
 
+[0.7.0]: https://github.com/datajoint/element-interface/releases/tag/0.7.0
 [0.6.0]: https://github.com/datajoint/element-interface/releases/tag/0.6.0
 [0.5.4]: https://github.com/datajoint/element-interface/releases/tag/0.5.4
 [0.5.3]: https://github.com/datajoint/element-interface/releases/tag/0.5.3

From 697e804d4a289c934d4310db440bb5b28bb37ca8 Mon Sep 17 00:00:00 2001
From: Thinh Nguyen <thinh@vathes.com>
Date: Tue, 5 Sep 2023 22:16:28 -0500
Subject: [PATCH 16/64] add `validation` argument to dandi upload

---
 element_interface/dandi.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/element_interface/dandi.py b/element_interface/dandi.py
index 078e58a..22486b3 100644
--- a/element_interface/dandi.py
+++ b/element_interface/dandi.py
@@ -12,6 +12,7 @@ def upload_to_dandi(
     api_key: str = None,
     sync: bool = False,
     existing: str = "refresh",
+    validation: str = "required",
     shell=True,  # without this param, subprocess interprets first arg as file/dir
 ):
     """Upload NWB files to DANDI Archive
@@ -27,6 +28,7 @@ def upload_to_dandi(
         sync (str, optional): If True, delete all files in archive that are not present
             in the local directory.
         existing (str, optional): see full description from `dandi upload --help`
+        validation (str, optional): [require|skip|ignore] see full description from `dandi upload --help`
     """
 
     working_directory = working_directory or os.path.curdir
@@ -84,4 +86,5 @@ def upload_to_dandi(
         dandi_instance="dandi-staging" if staging else "dandi",
         existing=existing,
         sync=sync,
+        validation=validation,
     )

From 7e5be44831668d408d7aebc626398d8658308dd0 Mon Sep 17 00:00:00 2001
From: kushalbakshi <kushalbakshi@gmail.com>
Date: Wed, 13 Sep 2023 16:30:58 -0500
Subject: [PATCH 17/64] Check for z_fields in 2nd cycle of acqusition

---
 element_interface/prairie_view_loader.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/element_interface/prairie_view_loader.py b/element_interface/prairie_view_loader.py
index 1a05139..7a02949 100644
--- a/element_interface/prairie_view_loader.py
+++ b/element_interface/prairie_view_loader.py
@@ -205,7 +205,7 @@ def _extract_prairieview_metadata(xml_filepath: str):
 
     if (
         xml_root.find(
-            ".//Sequence/[@cycle='1']/Frame/PVStateShard/PVStateValue/[@key='positionCurrent']/SubindexedValues/[@index='ZAxis']"
+            ".//Sequence/[@cycle='2']/Frame/PVStateShard/PVStateValue/[@key='positionCurrent']/SubindexedValues/[@index='ZAxis']"
         )
         is None
     ):
@@ -232,7 +232,7 @@ def _extract_prairieview_metadata(xml_filepath: str):
         n_depths = len(plane_indices)
 
         z_controllers = xml_root.findall(
-            ".//Sequence/[@cycle='1']/Frame/[@index='1']/PVStateShard/PVStateValue/[@key='positionCurrent']/SubindexedValues/[@index='ZAxis']/SubindexedValue"
+            ".//Sequence/[@cycle='2']/Frame/[@index='1']/PVStateShard/PVStateValue/[@key='positionCurrent']/SubindexedValues/[@index='ZAxis']/SubindexedValue"
         )
 
         # If more than one Z-axis controllers are found,
@@ -241,13 +241,13 @@ def _extract_prairieview_metadata(xml_filepath: str):
         if len(z_controllers) > 1:
             z_repeats = []
             for controller in xml_root.findall(
-                ".//Sequence/[@cycle='1']/Frame/[@index='1']/PVStateShard/PVStateValue/[@key='positionCurrent']/SubindexedValues/[@index='ZAxis']/"
+                ".//Sequence/[@cycle='2']/Frame/[@index='1']/PVStateShard/PVStateValue/[@key='positionCurrent']/SubindexedValues/[@index='ZAxis']/"
             ):
                 z_repeats.append(
                     [
                         float(z.attrib.get("value"))
                         for z in xml_root.findall(
-                            ".//Sequence/[@cycle='1']/Frame/PVStateShard/PVStateValue/[@key='positionCurrent']/SubindexedValues/[@index='ZAxis']/SubindexedValue/[@subindex='{0}']".format(
+                            ".//Sequence/[@cycle='2']/Frame/PVStateShard/PVStateValue/[@key='positionCurrent']/SubindexedValues/[@index='ZAxis']/SubindexedValue/[@subindex='{0}']".format(
                                 controller.attrib.get("subindex")
                             )
                         )
@@ -267,7 +267,7 @@ def _extract_prairieview_metadata(xml_filepath: str):
             z_fields = [
                 z.attrib.get("value")
                 for z in xml_root.findall(
-                    ".//Sequence/[@cycle='1']/Frame/PVStateShard/PVStateValue/[@key='positionCurrent']/SubindexedValues/[@index='ZAxis']/SubindexedValue/[@subindex='0']"
+                    ".//Sequence/[@cycle='2']/Frame/PVStateShard/PVStateValue/[@key='positionCurrent']/SubindexedValues/[@index='ZAxis']/SubindexedValue/[@subindex='0']"
                 )
             ]
 

From b374a41c5029af32e3d1dda1f89460324ef881e9 Mon Sep 17 00:00:00 2001
From: Thinh Nguyen <thinh@datajoint.com>
Date: Thu, 12 Oct 2023 15:48:15 -0500
Subject: [PATCH 18/64] passing as argument to `fit_file`

---
 element_interface/run_caiman.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/element_interface/run_caiman.py b/element_interface/run_caiman.py
index 6cb1559..91913c4 100644
--- a/element_interface/run_caiman.py
+++ b/element_interface/run_caiman.py
@@ -35,6 +35,13 @@ def run_caiman(
     parameters["fnames"] = file_paths
     parameters["fr"] = sampling_rate
 
+    if "indicies" in parameters:
+        indices = params.pop("indicies")
+        indicies = slice(*indices[0]), slice(*indices[1])
+        parameters['motion'] = {**parameters.get('motion', {}), "indicies": indicies}
+    else:
+        indicies = None
+
     opts = params.CNMFParams(params_dict=parameters)
 
     c, dview, n_processes = cm.cluster.setup_cluster(
@@ -45,6 +52,7 @@ def run_caiman(
         cnm = CNMF(n_processes, params=opts, dview=dview)
         cnmf_output, mc_output = cnm.fit_file(
             motion_correct=True,
+            indices=indicies,
             include_eval=True,
             output_dir=output_dir,
             return_mc=True,

From a558909704bb85418a7d0087e387ca7d2b6b792a Mon Sep 17 00:00:00 2001
From: kushalbakshi <kushalbakshi@gmail.com>
Date: Mon, 16 Oct 2023 14:36:30 -0500
Subject: [PATCH 19/64] Fix typo indicies -> indices

---
 element_interface/run_caiman.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/element_interface/run_caiman.py b/element_interface/run_caiman.py
index 91913c4..a38c6db 100644
--- a/element_interface/run_caiman.py
+++ b/element_interface/run_caiman.py
@@ -35,12 +35,12 @@ def run_caiman(
     parameters["fnames"] = file_paths
     parameters["fr"] = sampling_rate
 
-    if "indicies" in parameters:
-        indices = params.pop("indicies")
-        indicies = slice(*indices[0]), slice(*indices[1])
-        parameters['motion'] = {**parameters.get('motion', {}), "indicies": indicies}
+    if "indices" in parameters:
+        indices = params.pop("indices")
+        indices = slice(*indices[0]), slice(*indices[1])
+        parameters['motion'] = {**parameters.get('motion', {}), "indices": indices}
     else:
-        indicies = None
+        indices = None
 
     opts = params.CNMFParams(params_dict=parameters)
 
@@ -52,7 +52,7 @@ def run_caiman(
         cnm = CNMF(n_processes, params=opts, dview=dview)
         cnmf_output, mc_output = cnm.fit_file(
             motion_correct=True,
-            indices=indicies,
+            indices=indices,
             include_eval=True,
             output_dir=output_dir,
             return_mc=True,

From 23e14b448e019717e619bb9cefd23234392edc10 Mon Sep 17 00:00:00 2001
From: kushalbakshi <kushalbakshi@gmail.com>
Date: Mon, 16 Oct 2023 17:18:32 -0500
Subject: [PATCH 20/64] Update params -> parameters

---
 element_interface/run_caiman.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/element_interface/run_caiman.py b/element_interface/run_caiman.py
index a38c6db..60d2112 100644
--- a/element_interface/run_caiman.py
+++ b/element_interface/run_caiman.py
@@ -36,9 +36,9 @@ def run_caiman(
     parameters["fr"] = sampling_rate
 
     if "indices" in parameters:
-        indices = params.pop("indices")
+        indices = parameters.pop("indices")
         indices = slice(*indices[0]), slice(*indices[1])
-        parameters['motion'] = {**parameters.get('motion', {}), "indices": indices}
+        parameters["motion"] = {**parameters.get("motion", {}), "indices": indices}
     else:
         indices = None
 

From fff8aba3d22ff97215ea19a14a57dab9db2e23de Mon Sep 17 00:00:00 2001
From: kushalbakshi <kushalbakshi@gmail.com>
Date: Mon, 23 Oct 2023 15:34:30 -0500
Subject: [PATCH 21/64] Set `indices=None` in `fit_file()` + comments for
 clarity

---
 element_interface/run_caiman.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/element_interface/run_caiman.py b/element_interface/run_caiman.py
index 91913c4..b9ec3ea 100644
--- a/element_interface/run_caiman.py
+++ b/element_interface/run_caiman.py
@@ -36,11 +36,11 @@ def run_caiman(
     parameters["fr"] = sampling_rate
 
     if "indicies" in parameters:
-        indices = params.pop("indicies")
+        indices = params.pop(
+            "indicies"
+        )  # Indices that restrict FOV for motion correction.
         indicies = slice(*indices[0]), slice(*indices[1])
-        parameters['motion'] = {**parameters.get('motion', {}), "indicies": indicies}
-    else:
-        indicies = None
+        parameters["motion"] = {**parameters.get("motion", {}), "indicies": indicies}
 
     opts = params.CNMFParams(params_dict=parameters)
 
@@ -52,7 +52,7 @@ def run_caiman(
         cnm = CNMF(n_processes, params=opts, dview=dview)
         cnmf_output, mc_output = cnm.fit_file(
             motion_correct=True,
-            indices=indicies,
+            indices=None,  # Indices defined here restrict FOV for segmentation. `None` uses the full image for segmentation.
             include_eval=True,
             output_dir=output_dir,
             return_mc=True,

From 1efb4d2a223def6022ee5c0c7606c33417525450 Mon Sep 17 00:00:00 2001
From: kushalbakshi <kushalbakshi@gmail.com>
Date: Mon, 23 Oct 2023 15:39:56 -0500
Subject: [PATCH 22/64] Fix typos

---
 element_interface/run_caiman.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/element_interface/run_caiman.py b/element_interface/run_caiman.py
index b9ec3ea..db6a059 100644
--- a/element_interface/run_caiman.py
+++ b/element_interface/run_caiman.py
@@ -35,12 +35,12 @@ def run_caiman(
     parameters["fnames"] = file_paths
     parameters["fr"] = sampling_rate
 
-    if "indicies" in parameters:
-        indices = params.pop(
-            "indicies"
+    if "indices" in parameters:
+        indices = parameters.pop(
+            "indices"
         )  # Indices that restrict FOV for motion correction.
-        indicies = slice(*indices[0]), slice(*indices[1])
-        parameters["motion"] = {**parameters.get("motion", {}), "indicies": indicies}
+        indices = slice(*indices[0]), slice(*indices[1])
+        parameters["motion"] = {**parameters.get("motion", {}), "indices": indices}
 
     opts = params.CNMFParams(params_dict=parameters)
 

From da0ef6bc872816024a609104f1ca84344f519d9f Mon Sep 17 00:00:00 2001
From: Thinh Nguyen <thinh@datajoint.com>
Date: Tue, 30 Jan 2024 15:02:14 -0600
Subject: [PATCH 23/64] fix(prairie_view_loader): fix file search for
 single-plane dataset

---
 element_interface/prairie_view_loader.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/element_interface/prairie_view_loader.py b/element_interface/prairie_view_loader.py
index 1a05139..68abe37 100644
--- a/element_interface/prairie_view_loader.py
+++ b/element_interface/prairie_view_loader.py
@@ -73,9 +73,11 @@ def get_prairieview_filenames(
             assert (
                 channel in self.meta["channels"]
             ), f"Invalid 'channel' - Channels: {self.meta['channels']}"
+        
 
+        plane_search_string = f"[@index='{plane_idx}']/" if self.meta["num_planes"] > 1 else "" 
         frames = self._xml_root.findall(
-            f".//Sequence/Frame/[@index='{plane_idx}']/File/[@channel='{channel}']"
+            f".//Sequence/Frame/{plane_search_string}File/[@channel='{channel}']"
         )
 
         fnames = [f.attrib["filename"] for f in frames]

From eb4ac75cadd0e2dea56b1641af784bb51da66d79 Mon Sep 17 00:00:00 2001
From: Thinh Nguyen <thinh@vathes.com>
Date: Thu, 1 Feb 2024 13:31:38 -0600
Subject: [PATCH 24/64] fix(prairie_view_loader): fix handling of `num_frames`
 and plane search for single/multi plane dataset

---
 element_interface/prairie_view_loader.py | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/element_interface/prairie_view_loader.py b/element_interface/prairie_view_loader.py
index c5d93bc..5eb8dcd 100644
--- a/element_interface/prairie_view_loader.py
+++ b/element_interface/prairie_view_loader.py
@@ -36,6 +36,13 @@ def __init__(self, prairieview_dir: str):
     def meta(self):
         if self._meta is None:
             self._meta = _extract_prairieview_metadata(self.xml_file)
+            # adjust for the different definition of "frames"
+            # from the ome meta - "frame" refers to an image at a given scanning depth, time step combination
+            # in the imaging pipeline - "frame" refers to video frames - i.e. time steps
+            num_frames = self._meta.pop("num_frames") / self._meta["num_planes"]
+
+            self._meta["num_frames"] = int(num_frames)
+
         return self._meta
 
     def get_prairieview_filenames(
@@ -73,11 +80,14 @@ def get_prairieview_filenames(
             assert (
                 channel in self.meta["channels"]
             ), f"Invalid 'channel' - Channels: {self.meta['channels']}"
-        
 
-        plane_search_string = f"[@index='{plane_idx}']/" if self.meta["num_planes"] > 1 else "" 
+        # single-plane ome.tif does not have "@index" under Frame to search for
+        plane_search = f"/[@index='{plane_idx}']" if self.meta["num_planes"] > 1 else ""
+        # ome.tif does have "@channel" under File regardless of single or multi channel
+        channel_search = f"/[@channel='{channel}']"
+
         frames = self._xml_root.findall(
-            f".//Sequence/Frame/{plane_search_string}File/[@channel='{channel}']"
+            f".//Sequence/Frame{plane_search}/File{channel_search}"
         )
 
         fnames = [f.attrib["filename"] for f in frames]

From 66fa78806b4942cbc12e08c7e90d0391d4a49b79 Mon Sep 17 00:00:00 2001
From: kushalbakshi <kushalbakshi@gmail.com>
Date: Fri, 16 Feb 2024 10:13:28 -0600
Subject: [PATCH 25/64] Bugfix - remove hardcoded `n_fields` from
 `prairie_view_loader.py`

---
 CHANGELOG.md                             | 1 +
 element_interface/prairie_view_loader.py | 3 +--
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 54478f9..f4256fb 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,7 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and
 
 + Add - multi-plane `caiman_loader.py` to process multi-plane tiffs
 + Update - DANDI upload utility
++ Fix - `n_fields` == 1 -> `n_fields` == `n_depths` in `prairie_view_loader.py`
 
 ## [0.6.1] - 2023-08-02
 
diff --git a/element_interface/prairie_view_loader.py b/element_interface/prairie_view_loader.py
index 5eb8dcd..46bb266 100644
--- a/element_interface/prairie_view_loader.py
+++ b/element_interface/prairie_view_loader.py
@@ -155,7 +155,6 @@ def _extract_prairieview_metadata(xml_filepath: str):
 
     bidirectional_scan = False  # Does not support bidirectional
     roi = 0
-    n_fields = 1  # Always contains 1 field
     recording_start_time = xml_root.find(".//Sequence/[@cycle='1']").attrib.get("time")
 
     # Get all channels and find unique values
@@ -288,7 +287,7 @@ def _extract_prairieview_metadata(xml_filepath: str):
         ), "Number of z fields does not match number of depths."
 
     metainfo = dict(
-        num_fields=n_fields,
+        num_fields=n_depths,
         num_channels=n_channels,
         num_planes=n_depths,
         num_frames=n_frames,

From 46d8df4858d3a90fea850fe52b35e0beec4506c3 Mon Sep 17 00:00:00 2001
From: kushalbakshi <kushalbakshi@gmail.com>
Date: Mon, 19 Feb 2024 17:23:06 -0600
Subject: [PATCH 26/64] Add meaningful exception error in
 `write_single_bigtiff()`

---
 element_interface/prairie_view_loader.py | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/element_interface/prairie_view_loader.py b/element_interface/prairie_view_loader.py
index 46bb266..153e38e 100644
--- a/element_interface/prairie_view_loader.py
+++ b/element_interface/prairie_view_loader.py
@@ -119,13 +119,16 @@ def write_single_bigtiff(
                 output_tiff_fullpath,
                 bigtiff=True,
             ) as tiff_writer:
-                for input_file in tiff_names:
-                    with tifffile.TiffFile(self.prairieview_dir / input_file) as tffl:
-                        assert len(tffl.pages) == 1
-                        tiff_writer.write(
-                            tffl.pages[0].asarray(),
-                            metadata={"axes": "YX", "'fps'": self.meta["frame_rate"]},
-                        )
+                try:
+                    for input_file in tiff_names:
+                        with tifffile.TiffFile(self.prairieview_dir / input_file) as tffl:
+                            assert len(tffl.pages) == 1
+                            tiff_writer.write(
+                                tffl.pages[0].asarray(),
+                                metadata={"axes": "YX", "'fps'": self.meta["frame_rate"]},
+                            )
+                except Exception as e:
+                    raise f"Errow creating big tiff for {input_file}: {e}"
         else:
             combined_data = []
             for input_file in tiff_names:

From 42269100ba361018dc42576a43ff7f56f155cb78 Mon Sep 17 00:00:00 2001
From: Kushal Bakshi <52367253+kushalbakshi@users.noreply.github.com>
Date: Tue, 20 Feb 2024 08:40:04 -0600
Subject: [PATCH 27/64] Update element_interface/prairie_view_loader.py

Co-authored-by: Thinh Nguyen <thinh@datajoint.com>
---
 element_interface/prairie_view_loader.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/element_interface/prairie_view_loader.py b/element_interface/prairie_view_loader.py
index 153e38e..3f28f20 100644
--- a/element_interface/prairie_view_loader.py
+++ b/element_interface/prairie_view_loader.py
@@ -128,7 +128,7 @@ def write_single_bigtiff(
                                 metadata={"axes": "YX", "'fps'": self.meta["frame_rate"]},
                             )
                 except Exception as e:
-                    raise f"Errow creating big tiff for {input_file}: {e}"
+                    raise f"Error in processing tiff file {input_file}: {e}"
         else:
             combined_data = []
             for input_file in tiff_names:

From 84b4b1e945ab2ac7e41309290ed32522ffc750d9 Mon Sep 17 00:00:00 2001
From: kushalbakshi <kushalbakshi@gmail.com>
Date: Tue, 20 Feb 2024 17:47:31 -0600
Subject: [PATCH 28/64] Catch expection for caiman compatible files in
 prairie_view_loader

---
 element_interface/prairie_view_loader.py | 21 +++++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)

diff --git a/element_interface/prairie_view_loader.py b/element_interface/prairie_view_loader.py
index 3f28f20..38dfbe3 100644
--- a/element_interface/prairie_view_loader.py
+++ b/element_interface/prairie_view_loader.py
@@ -121,20 +121,29 @@ def write_single_bigtiff(
             ) as tiff_writer:
                 try:
                     for input_file in tiff_names:
-                        with tifffile.TiffFile(self.prairieview_dir / input_file) as tffl:
+                        with tifffile.TiffFile(
+                            self.prairieview_dir / input_file
+                        ) as tffl:
                             assert len(tffl.pages) == 1
                             tiff_writer.write(
                                 tffl.pages[0].asarray(),
-                                metadata={"axes": "YX", "'fps'": self.meta["frame_rate"]},
+                                metadata={
+                                    "axes": "YX",
+                                    "'fps'": self.meta["frame_rate"],
+                                },
                             )
                 except Exception as e:
                     raise f"Error in processing tiff file {input_file}: {e}"
         else:
             combined_data = []
-            for input_file in tiff_names:
-                with tifffile.TiffFile(self.prairieview_dir / input_file) as tffl:
-                    assert len(tffl.pages) == 1
-                    combined_data.append(tffl.pages[0].asarray())
+            try:
+                for input_file in tiff_names:
+                    with tifffile.TiffFile(self.prairieview_dir / input_file) as tffl:
+                        assert len(tffl.pages) == 1
+                        combined_data.append(tffl.pages[0].asarray())
+            except Exception as e:
+                raise f"Error in processing tiff file {input_file}: {e}"
+
             combined_data = np.dstack(combined_data).transpose(
                 2, 0, 1
             )  # (frame x height x width)

From 6747d8a1ad8198f8a3cc6c4294e6c067a8645e1c Mon Sep 17 00:00:00 2001
From: kushalbakshi <kushalbakshi@gmail.com>
Date: Wed, 21 Feb 2024 14:51:29 -0600
Subject: [PATCH 29/64] Add `Exception` to `raise` statements

---
 element_interface/prairie_view_loader.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/element_interface/prairie_view_loader.py b/element_interface/prairie_view_loader.py
index 38dfbe3..e6dddc9 100644
--- a/element_interface/prairie_view_loader.py
+++ b/element_interface/prairie_view_loader.py
@@ -133,7 +133,7 @@ def write_single_bigtiff(
                                 },
                             )
                 except Exception as e:
-                    raise f"Error in processing tiff file {input_file}: {e}"
+                    raise Exception(f"Error in processing tiff file {input_file}: {e}")
         else:
             combined_data = []
             try:
@@ -142,7 +142,7 @@ def write_single_bigtiff(
                         assert len(tffl.pages) == 1
                         combined_data.append(tffl.pages[0].asarray())
             except Exception as e:
-                raise f"Error in processing tiff file {input_file}: {e}"
+                raise Exception(f"Error in processing tiff file {input_file}: {e}")
 
             combined_data = np.dstack(combined_data).transpose(
                 2, 0, 1

From 11a1a0f42838d335f291419bf9816b253b0337e7 Mon Sep 17 00:00:00 2001
From: Thinh Nguyen <thinh@datajoint.com>
Date: Wed, 20 Mar 2024 12:57:44 -0500
Subject: [PATCH 30/64] Revert "Merge branch 'main' into dev_memoized_results"

This reverts commit 012e83857c13a96d36109faa16c46aebdbddc9f0, reversing
changes made to b3f68292b7c7f46563d1b20525a21401044cf2e4.
---
 element_interface/utils.py | 69 +++++++++++++++++++++++++++++++++++++-
 1 file changed, 68 insertions(+), 1 deletion(-)

diff --git a/element_interface/utils.py b/element_interface/utils.py
index 14d4eee..c3832f4 100644
--- a/element_interface/utils.py
+++ b/element_interface/utils.py
@@ -5,7 +5,9 @@
 import pathlib
 import sys
 import uuid
-
+import json
+import pickle
+from datetime import datetime
 from datajoint.utils import to_camel_case
 
 logger = logging.getLogger("datajoint")
@@ -187,3 +189,68 @@ def __exit__(self, *args):
         logger.setLevel(self.prev_log_level)
         sys.stdout.close()
         sys.stdout = self._original_stdout
+
+
+def memoized_result(parameters: dict, output_directory: str):
+    """
+    This is a decorator factory designed to cache the results of a function based on its input parameters and the state of the output directory.
+     If the function is called with the same parameters and the output files in the directory remain unchanged,
+      it returns the cached results; otherwise, it executes the function and caches the new results along with metadata.
+    Conditions for robust usage:
+        - the "output_directory" is to store exclusively the resulting files generated by this function call only, not a shared space with other functions/processes
+        - the "parameters" passed to the decorator captures the true and uniqueness of the arguments to be used in the decorated function call
+    Args:
+        parameters: parameters that would identify a unique function call
+        output_directory: directory location for the output files
+
+    Returns: a decorator to enable a function call to memoize/cached the resulting files
+    """
+
+    def decorator(func):
+        def wrapped(*args, **kwargs):
+            output_dir = _to_Path(output_directory)
+            input_hash = dict_to_uuid(parameters)
+            input_hash_fp = output_dir / f".{input_hash}.json"
+            # check if results already exist (from previous identical run)
+            output_dir_files_hash = dict_to_uuid(
+                {
+                    f.relative_to(output_dir).as_posix(): f.stat().st_size
+                    for f in output_dir.rglob("*")
+                    if f.name != f".{input_hash}.json"
+                }
+            )
+            if input_hash_fp.exists():
+                with open(input_hash_fp, "r") as f:
+                    meta = json.load(f)
+                if str(output_dir_files_hash) == meta["output_dir_files_hash"]:
+                    logger.info(f"Existing results found, skip '{func.__name__}'")
+                    with open(output_dir / f".{input_hash}_results.pickle", "rb") as f:
+                        results = pickle.load(f)
+                    return results
+            # no results - trigger the run
+            logger.info(f"No existing results found, calling '{func.__name__}'")
+            start_time = datetime.utcnow()
+            results = func(*args, **kwargs)
+
+            with open(output_dir / f".{input_hash}_results.pickle", "wb") as f:
+                pickle.dump(results, f, protocol=pickle.HIGHEST_PROTOCOL)
+
+            meta = {
+                "output_dir_files_hash": dict_to_uuid(
+                    {
+                        f.relative_to(output_dir).as_posix(): f.stat().st_size
+                        for f in output_dir.rglob("*")
+                        if f.name != f".{input_hash}.json"
+                    }
+                ),
+                "start_time": start_time,
+                "completion_time": datetime.utcnow(),
+            }
+            with open(input_hash_fp, "w") as f:
+                json.dump(meta, f, default=str)
+
+            return results
+
+        return wrapped
+
+    return decorator

From f4d84780ceb11bfbb3ec5dc53ec5331c1a9be037 Mon Sep 17 00:00:00 2001
From: Thinh Nguyen <thinh@datajoint.com>
Date: Wed, 20 Mar 2024 15:16:23 -0500
Subject: [PATCH 31/64] format: minor reformatting of the docstring

---
 element_interface/utils.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/element_interface/utils.py b/element_interface/utils.py
index c3832f4..14da994 100644
--- a/element_interface/utils.py
+++ b/element_interface/utils.py
@@ -196,14 +196,16 @@ def memoized_result(parameters: dict, output_directory: str):
     This is a decorator factory designed to cache the results of a function based on its input parameters and the state of the output directory.
      If the function is called with the same parameters and the output files in the directory remain unchanged,
       it returns the cached results; otherwise, it executes the function and caches the new results along with metadata.
-    Conditions for robust usage:
-        - the "output_directory" is to store exclusively the resulting files generated by this function call only, not a shared space with other functions/processes
-        - the "parameters" passed to the decorator captures the true and uniqueness of the arguments to be used in the decorated function call
+
     Args:
         parameters: parameters that would identify a unique function call
         output_directory: directory location for the output files
 
     Returns: a decorator to enable a function call to memoize/cached the resulting files
+
+    Conditions for robust usage:
+        - the "output_directory" is to store exclusively the resulting files generated by this function call only, not a shared space with other functions/processes
+        - the "parameters" passed to the decorator captures the true and uniqueness of the arguments to be used in the decorated function call
     """
 
     def decorator(func):

From deff0fa76bacf6c43a3cf9d0aebe373989b7ca68 Mon Sep 17 00:00:00 2001
From: Thinh Nguyen <thinh@datajoint.com>
Date: Thu, 18 Apr 2024 10:44:38 -0500
Subject: [PATCH 32/64] fix{prairie_view_loader): bugfix framerate

---
 element_interface/prairie_view_loader.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/element_interface/prairie_view_loader.py b/element_interface/prairie_view_loader.py
index e6dddc9..6dcda90 100644
--- a/element_interface/prairie_view_loader.py
+++ b/element_interface/prairie_view_loader.py
@@ -39,9 +39,9 @@ def meta(self):
             # adjust for the different definition of "frames"
             # from the ome meta - "frame" refers to an image at a given scanning depth, time step combination
             # in the imaging pipeline - "frame" refers to video frames - i.e. time steps
-            num_frames = self._meta.pop("num_frames") / self._meta["num_planes"]
-
-            self._meta["num_frames"] = int(num_frames)
+            num_frames = int(self._meta.pop("num_frames") / self._meta["num_planes"])
+            self._meta["num_frames"] = num_frames
+            self._meta["frame_rate"] = num_frames / self._meta["scan_duration"]
 
         return self._meta
 
@@ -177,9 +177,9 @@ def _extract_prairieview_metadata(xml_filepath: str):
     channels = set(channel_list)
     n_channels = len(channels)
     n_frames = len(xml_root.findall(".//Sequence/Frame"))
-    framerate = 1 / float(
+    frame_period = float(
         xml_root.findall('.//PVStateValue/[@key="framePeriod"]')[0].attrib.get("value")
-    )  # rate = 1/framePeriod
+    )
 
     usec_per_line = (
         float(
@@ -307,7 +307,7 @@ def _extract_prairieview_metadata(xml_filepath: str):
         x_pos=None,
         y_pos=None,
         z_pos=None,
-        frame_rate=framerate,
+        frame_period=frame_period,
         bidirectional=bidirectional_scan,
         bidirectional_z=bidirection_z,
         scan_datetime=scan_datetime,

From 3ad3ed44dd093cc1d680dde23e2e4ed19dea3dc1 Mon Sep 17 00:00:00 2001
From: kushalbakshi <kushalbakshi@gmail.com>
Date: Tue, 23 Apr 2024 16:33:14 -0500
Subject: [PATCH 33/64] Check for bruker-generated multi_tiff

---
 element_interface/prairie_view_loader.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/element_interface/prairie_view_loader.py b/element_interface/prairie_view_loader.py
index 6dcda90..9854a22 100644
--- a/element_interface/prairie_view_loader.py
+++ b/element_interface/prairie_view_loader.py
@@ -167,6 +167,7 @@ def _extract_prairieview_metadata(xml_filepath: str):
 
     bidirectional_scan = False  # Does not support bidirectional
     roi = 0
+    multi_tiff = xml_root.find(".//Sequence/Frame/File/[@page]") is not None
     recording_start_time = xml_root.find(".//Sequence/[@cycle='1']").attrib.get("time")
 
     # Get all channels and find unique values
@@ -310,6 +311,7 @@ def _extract_prairieview_metadata(xml_filepath: str):
         frame_period=frame_period,
         bidirectional=bidirectional_scan,
         bidirectional_z=bidirection_z,
+        multi_tiff=multi_tiff,
         scan_datetime=scan_datetime,
         usecs_per_line=usec_per_line,
         scan_duration=total_scan_duration,

From b5803e7216d7cfa12d36fc8cc76ecf37fdfd026b Mon Sep 17 00:00:00 2001
From: Thinh Nguyen <thinh@datajoint.com>
Date: Mon, 20 May 2024 14:49:42 -0500
Subject: [PATCH 34/64] feat: handle PrairieView new multi-page tif format

---
 element_interface/prairie_view_loader.py | 104 ++++++++++++++++-------
 1 file changed, 75 insertions(+), 29 deletions(-)

diff --git a/element_interface/prairie_view_loader.py b/element_interface/prairie_view_loader.py
index 6dcda90..f43f7ca 100644
--- a/element_interface/prairie_view_loader.py
+++ b/element_interface/prairie_view_loader.py
@@ -90,7 +90,7 @@ def get_prairieview_filenames(
             f".//Sequence/Frame{plane_search}/File{channel_search}"
         )
 
-        fnames = [f.attrib["filename"] for f in frames]
+        fnames = np.unique([f.attrib["filename"] for f in frames]).tolist()
         return fnames if not return_pln_chn else (fnames, plane_idx, channel)
 
     def write_single_bigtiff(
@@ -114,46 +114,90 @@ def write_single_bigtiff(
         if output_tiff_fullpath.exists() and not overwrite:
             return output_tiff_fullpath
 
-        if not caiman_compatible:
-            with tifffile.TiffWriter(
-                output_tiff_fullpath,
-                bigtiff=True,
-            ) as tiff_writer:
-                try:
-                    for input_file in tiff_names:
-                        with tifffile.TiffFile(
-                            self.prairieview_dir / input_file
-                        ) as tffl:
-                            assert len(tffl.pages) == 1
-                            tiff_writer.write(
-                                tffl.pages[0].asarray(),
-                                metadata={
-                                    "axes": "YX",
-                                    "'fps'": self.meta["frame_rate"],
-                                },
-                            )
-                except Exception as e:
-                    raise Exception(f"Error in processing tiff file {input_file}: {e}")
-        else:
-            combined_data = []
+        if self.meta["is_multipage"]:
+            # For multi-page tiff - the pages are organized as:
+            # (channel x slice x frame) - each page is (height x width)
+            # - TODO: verify this is the case for Bruker multi-page tiff
+            # This implementation is partially based on the reference code from `scanreader` package - https://github.com/atlab/scanreader
+            # See: https://github.com/atlab/scanreader/blob/2a021a85fca011c17e553d0e1c776998d3f2b2d8/scanreader/scans.py#L337
+            slice_step = self.meta["num_channels"]
+            frame_step = self.meta["num_channels"] * self.meta["num_planes"]
+            slice_idx = self.meta["plane_indices"].index(plane_idx)
+            channel_idx = self.meta["channels"].index(channel)
+
+            page_indices = [frame_idx * frame_step + slice_idx * slice_step + channel_idx
+                            for frame_idx in range(self.meta["num_frames"])]
+
+            combined_data = np.empty([self.meta["num_frames"],
+                                      self.meta["height_in_pixels"],
+                                      self.meta["width_in_pixels"]],
+                                     dtype=int)
+            start_page = 0
             try:
                 for input_file in tiff_names:
                     with tifffile.TiffFile(self.prairieview_dir / input_file) as tffl:
-                        assert len(tffl.pages) == 1
-                        combined_data.append(tffl.pages[0].asarray())
+                        # Get indices in this tiff file and in output array
+                        final_page_in_file = start_page + len(tffl.pages)
+                        is_page_in_file = lambda page: page in range(start_page, final_page_in_file)
+                        pages_in_file = filter(is_page_in_file, page_indices)
+                        file_indices = [page - start_page for page in pages_in_file]
+                        global_indices = [is_page_in_file(page) for page in page_indices]
+
+                        # Read from this tiff file (if needed)
+                        if len(file_indices) > 0:
+                            # this line looks a bit ugly but is memory efficient. Do not separate
+                            combined_data[global_indices] = tffl.asarray(key=file_indices)
+                        start_page += len(tffl.pages)
             except Exception as e:
                 raise Exception(f"Error in processing tiff file {input_file}: {e}")
 
-            combined_data = np.dstack(combined_data).transpose(
-                2, 0, 1
-            )  # (frame x height x width)
-
             tifffile.imwrite(
                 output_tiff_fullpath,
                 combined_data,
                 metadata={"axes": "TYX", "'fps'": self.meta["frame_rate"]},
                 bigtiff=True,
             )
+        else:
+            if not caiman_compatible:
+                with tifffile.TiffWriter(
+                    output_tiff_fullpath,
+                    bigtiff=True,
+                ) as tiff_writer:
+                    try:
+                        for input_file in tiff_names:
+                            with tifffile.TiffFile(
+                                self.prairieview_dir / input_file
+                            ) as tffl:
+                                assert len(tffl.pages) == 1
+                                tiff_writer.write(
+                                    tffl.pages[0].asarray(),
+                                    metadata={
+                                        "axes": "YX",
+                                        "'fps'": self.meta["frame_rate"],
+                                    },
+                                )
+                    except Exception as e:
+                        raise Exception(f"Error in processing tiff file {input_file}: {e}")
+            else:
+                combined_data = []
+                try:
+                    for input_file in tiff_names:
+                        with tifffile.TiffFile(self.prairieview_dir / input_file) as tffl:
+                            assert len(tffl.pages) == 1
+                            combined_data.append(tffl.pages[0].asarray())
+                except Exception as e:
+                    raise Exception(f"Error in processing tiff file {input_file}: {e}")
+
+                combined_data = np.dstack(combined_data).transpose(
+                    2, 0, 1
+                )  # (frame x height x width)
+
+                tifffile.imwrite(
+                    output_tiff_fullpath,
+                    combined_data,
+                    metadata={"axes": "TYX", "'fps'": self.meta["frame_rate"]},
+                    bigtiff=True,
+                )
 
         return output_tiff_fullpath
 
@@ -167,6 +211,7 @@ def _extract_prairieview_metadata(xml_filepath: str):
 
     bidirectional_scan = False  # Does not support bidirectional
     roi = 0
+    is_multipage = xml_root.find(".//Sequence/Frame/File/[@page]") is not None
     recording_start_time = xml_root.find(".//Sequence/[@cycle='1']").attrib.get("time")
 
     # Get all channels and find unique values
@@ -310,6 +355,7 @@ def _extract_prairieview_metadata(xml_filepath: str):
         frame_period=frame_period,
         bidirectional=bidirectional_scan,
         bidirectional_z=bidirection_z,
+        is_multipage=is_multipage,
         scan_datetime=scan_datetime,
         usecs_per_line=usec_per_line,
         scan_duration=total_scan_duration,

From f96d8be9309647a6f19a5378d9c59cdea88a9943 Mon Sep 17 00:00:00 2001
From: Thinh Nguyen <thinh@datajoint.com>
Date: Tue, 28 May 2024 12:11:08 -0500
Subject: [PATCH 35/64] update: minor var name change

---
 element_interface/utils.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/element_interface/utils.py b/element_interface/utils.py
index 14da994..2fc8ca9 100644
--- a/element_interface/utils.py
+++ b/element_interface/utils.py
@@ -191,14 +191,14 @@ def __exit__(self, *args):
         sys.stdout = self._original_stdout
 
 
-def memoized_result(parameters: dict, output_directory: str):
+def memoized_result(uniqueness_dict: dict, output_directory: str):
     """
     This is a decorator factory designed to cache the results of a function based on its input parameters and the state of the output directory.
      If the function is called with the same parameters and the output files in the directory remain unchanged,
       it returns the cached results; otherwise, it executes the function and caches the new results along with metadata.
 
     Args:
-        parameters: parameters that would identify a unique function call
+        uniqueness_dict: a dictionary that would identify a unique function call
         output_directory: directory location for the output files
 
     Returns: a decorator to enable a function call to memoize/cached the resulting files
@@ -211,7 +211,7 @@ def memoized_result(parameters: dict, output_directory: str):
     def decorator(func):
         def wrapped(*args, **kwargs):
             output_dir = _to_Path(output_directory)
-            input_hash = dict_to_uuid(parameters)
+            input_hash = dict_to_uuid(uniqueness_dict)
             input_hash_fp = output_dir / f".{input_hash}.json"
             # check if results already exist (from previous identical run)
             output_dir_files_hash = dict_to_uuid(

From 1f3ab21e0621b0a7293185b6d37a5e31548df4a7 Mon Sep 17 00:00:00 2001
From: kushalbakshi <kushalbakshi@gmail.com>
Date: Tue, 4 Jun 2024 16:30:08 -0400
Subject: [PATCH 36/64] Update(run_caiman.py) use single thread to avoid
 multiprocessing error

---
 element_interface/run_caiman.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/element_interface/run_caiman.py b/element_interface/run_caiman.py
index db6a059..5ca74d4 100644
--- a/element_interface/run_caiman.py
+++ b/element_interface/run_caiman.py
@@ -45,7 +45,7 @@ def run_caiman(
     opts = params.CNMFParams(params_dict=parameters)
 
     c, dview, n_processes = cm.cluster.setup_cluster(
-        backend="local", n_processes=None, single_thread=False
+        backend="local", n_processes=None, single_thread=True
     )
 
     try:

From 3fc7c17da83496e25d42daf4ed5a8ccd9158191a Mon Sep 17 00:00:00 2001
From: kushalbakshi <kushalbakshi@gmail.com>
Date: Tue, 4 Jun 2024 16:35:19 -0400
Subject: [PATCH 37/64] Fix(run_caiman.py): backend for cluster

---
 element_interface/run_caiman.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/element_interface/run_caiman.py b/element_interface/run_caiman.py
index 5ca74d4..8dd24c1 100644
--- a/element_interface/run_caiman.py
+++ b/element_interface/run_caiman.py
@@ -45,7 +45,7 @@ def run_caiman(
     opts = params.CNMFParams(params_dict=parameters)
 
     c, dview, n_processes = cm.cluster.setup_cluster(
-        backend="local", n_processes=None, single_thread=True
+        backend="ipyparallel", n_processes=None, single_thread=False
     )
 
     try:

From 1e06ac8b2ca536b6afc4d4fbfd1d6fcc9ad6bfe4 Mon Sep 17 00:00:00 2001
From: kushalbakshi <kushalbakshi@gmail.com>
Date: Tue, 4 Jun 2024 16:40:19 -0400
Subject: [PATCH 38/64] Revert backend to "multiprocessing"

---
 element_interface/run_caiman.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/element_interface/run_caiman.py b/element_interface/run_caiman.py
index 8dd24c1..e5fd18e 100644
--- a/element_interface/run_caiman.py
+++ b/element_interface/run_caiman.py
@@ -45,7 +45,7 @@ def run_caiman(
     opts = params.CNMFParams(params_dict=parameters)
 
     c, dview, n_processes = cm.cluster.setup_cluster(
-        backend="ipyparallel", n_processes=None, single_thread=False
+        backend="multiprocessing", n_processes=None, single_thread=False
     )
 
     try:

From 53502c0818d94e089ae00f71670867f81694c50b Mon Sep 17 00:00:00 2001
From: kushalbakshi <kushalbakshi@gmail.com>
Date: Tue, 4 Jun 2024 17:36:01 -0400
Subject: [PATCH 39/64] Fix(run_caiman.py) Remove "single_thread" argument from
 setup_cluster

---
 element_interface/run_caiman.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/element_interface/run_caiman.py b/element_interface/run_caiman.py
index e5fd18e..d45eda0 100644
--- a/element_interface/run_caiman.py
+++ b/element_interface/run_caiman.py
@@ -45,7 +45,7 @@ def run_caiman(
     opts = params.CNMFParams(params_dict=parameters)
 
     c, dview, n_processes = cm.cluster.setup_cluster(
-        backend="multiprocessing", n_processes=None, single_thread=False
+        backend="multiprocessing", n_processes=None
     )
 
     try:

From 647142ff1f03dcd613e37b0af392461bdc8ad52a Mon Sep 17 00:00:00 2001
From: kushalbakshi <kushalbakshi@gmail.com>
Date: Wed, 5 Jun 2024 12:28:32 -0400
Subject: [PATCH 40/64] Update(run_caiman.py) logic to handle caiman output
 files

---
 element_interface/run_caiman.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/element_interface/run_caiman.py b/element_interface/run_caiman.py
index d45eda0..2840aa9 100644
--- a/element_interface/run_caiman.py
+++ b/element_interface/run_caiman.py
@@ -1,6 +1,7 @@
-import pathlib
-
 import cv2
+import os
+import pathlib
+import shutil
 
 try:
     cv2.setNumThreads(0)
@@ -65,6 +66,11 @@ def run_caiman(
 
     cnmf_output_file = pathlib.Path(cnmf_output.mmap_file[:-4] + "hdf5")
     assert cnmf_output_file.exists()
-    assert cnmf_output_file.parent == pathlib.Path(output_dir)
+    output_files = os.listdir(cnmf_output_file.parent)
+    for output_file in output_files:
+        try:
+            shutil.move(output_file, output_dir)
+        except FileExistsError:
+            print(f"File {output_file.name} already exists in {output_dir}. Skipping.")
 
     _save_mc(mc_output, cnmf_output_file.as_posix(), parameters["is3D"])

From 12a79de71f747f58b716cb5faa95359802c25a43 Mon Sep 17 00:00:00 2001
From: kushalbakshi <kushalbakshi@gmail.com>
Date: Wed, 5 Jun 2024 12:37:36 -0400
Subject: [PATCH 41/64] Run shutil with full path

---
 element_interface/run_caiman.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/element_interface/run_caiman.py b/element_interface/run_caiman.py
index 2840aa9..4aabc2e 100644
--- a/element_interface/run_caiman.py
+++ b/element_interface/run_caiman.py
@@ -69,7 +69,7 @@ def run_caiman(
     output_files = os.listdir(cnmf_output_file.parent)
     for output_file in output_files:
         try:
-            shutil.move(output_file, output_dir)
+            shutil.move(cnmf_output_file.parent / output_file, output_dir)
         except FileExistsError:
             print(f"File {output_file.name} already exists in {output_dir}. Skipping.")
 

From f78acfc50404bc8a36a65051586b69ea6694caa2 Mon Sep 17 00:00:00 2001
From: kushalbakshi <kushalbakshi@gmail.com>
Date: Wed, 5 Jun 2024 12:45:10 -0400
Subject: [PATCH 42/64] Copy files instead of move out of temp_dir

---
 element_interface/run_caiman.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/element_interface/run_caiman.py b/element_interface/run_caiman.py
index 4aabc2e..d45765c 100644
--- a/element_interface/run_caiman.py
+++ b/element_interface/run_caiman.py
@@ -69,7 +69,7 @@ def run_caiman(
     output_files = os.listdir(cnmf_output_file.parent)
     for output_file in output_files:
         try:
-            shutil.move(cnmf_output_file.parent / output_file, output_dir)
+            shutil.copy(cnmf_output_file.parent / output_file, output_dir)
         except FileExistsError:
             print(f"File {output_file.name} already exists in {output_dir}. Skipping.")
 

From 843ac6dfb54a751e7c01ab1f1872488fc470f5fc Mon Sep 17 00:00:00 2001
From: kushalbakshi <kushalbakshi@gmail.com>
Date: Wed, 5 Jun 2024 18:43:21 -0400
Subject: [PATCH 43/64] Fix(caiman_loader.py): pass generator obj as list to
 np.dstack

---
 element_interface/caiman_loader.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/element_interface/caiman_loader.py b/element_interface/caiman_loader.py
index a87d9eb..e245abd 100644
--- a/element_interface/caiman_loader.py
+++ b/element_interface/caiman_loader.py
@@ -293,8 +293,10 @@ def _get_image(self, img_type):
             )
         else:
             img_ = np.dstack(
-                pln_cm.motion_correction[img_type][...]
-                for pln_cm in self.planes.values()
+                [
+                    pln_cm.motion_correction[img_type][...]
+                    for pln_cm in self.planes.values()
+                ]
             )
         return img_
 

From 50e89776fad2a2bc83a3453daabead117bbabce8 Mon Sep 17 00:00:00 2001
From: kushalbakshi <kushalbakshi@gmail.com>
Date: Thu, 20 Jun 2024 14:17:57 -0400
Subject: [PATCH 44/64] Use memmap arrays to generate bigtiffs

---
 element_interface/prairie_view_loader.py | 41 +++++++++++++++---------
 1 file changed, 25 insertions(+), 16 deletions(-)

diff --git a/element_interface/prairie_view_loader.py b/element_interface/prairie_view_loader.py
index f43f7ca..77243b1 100644
--- a/element_interface/prairie_view_loader.py
+++ b/element_interface/prairie_view_loader.py
@@ -179,25 +179,34 @@ def write_single_bigtiff(
                     except Exception as e:
                         raise Exception(f"Error in processing tiff file {input_file}: {e}")
             else:
-                combined_data = []
                 try:
-                    for input_file in tiff_names:
-                        with tifffile.TiffFile(self.prairieview_dir / input_file) as tffl:
-                            assert len(tffl.pages) == 1
-                            combined_data.append(tffl.pages[0].asarray())
+                    with tifffile.TiffWriter(output_tiff_fullpath, bigtiff=True) as tif_writer:
+                        for input_file in tiff_names:
+                            with tifffile.TiffFile(self.prairieview_dir / input_file, mode='r') as tffl:
+                                assert len(tffl.pages) == 1
+                                data = tffl.pages[0].asarray(out='memmap')  # Use memory-mapped array
+                                tif_writer.write(data, metadata={"axes": "TYX", "fps": self.meta["frame_rate"]})
                 except Exception as e:
                     raise Exception(f"Error in processing tiff file {input_file}: {e}")
-
-                combined_data = np.dstack(combined_data).transpose(
-                    2, 0, 1
-                )  # (frame x height x width)
-
-                tifffile.imwrite(
-                    output_tiff_fullpath,
-                    combined_data,
-                    metadata={"axes": "TYX", "'fps'": self.meta["frame_rate"]},
-                    bigtiff=True,
-                )
+                # combined_data = []
+                # try:
+                #     for input_file in tiff_names:
+                #         with tifffile.TiffFile(self.prairieview_dir / input_file) as tffl:
+                #             assert len(tffl.pages) == 1
+                #             combined_data.append(tffl.pages[0].asarray())
+                # except Exception as e:
+                #     raise Exception(f"Error in processing tiff file {input_file}: {e}")
+
+                # combined_data = np.dstack(combined_data).transpose(
+                #     2, 0, 1
+                # )  # (frame x height x width)
+
+                # tifffile.imwrite(
+                #     output_tiff_fullpath,
+                #     combined_data,
+                #     metadata={"axes": "TYX", "'fps'": self.meta["frame_rate"]},
+                #     bigtiff=True,
+                # )
 
         return output_tiff_fullpath
 

From 24d8afa4b64ef98c29ee09e5c0cf74069ccecc5a Mon Sep 17 00:00:00 2001
From: kushalbakshi <kushalbakshi@gmail.com>
Date: Thu, 20 Jun 2024 15:46:02 -0400
Subject: [PATCH 45/64] Fix tiff file generation for caiman compatability

---
 element_interface/prairie_view_loader.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/element_interface/prairie_view_loader.py b/element_interface/prairie_view_loader.py
index 77243b1..a29f2f8 100644
--- a/element_interface/prairie_view_loader.py
+++ b/element_interface/prairie_view_loader.py
@@ -182,10 +182,16 @@ def write_single_bigtiff(
                 try:
                     with tifffile.TiffWriter(output_tiff_fullpath, bigtiff=True) as tif_writer:
                         for input_file in tiff_names:
-                            with tifffile.TiffFile(self.prairieview_dir / input_file, mode='r') as tffl:
+                            with tifffile.TiffFile(self.prairieview_dir / input_file) as tffl:
                                 assert len(tffl.pages) == 1
-                                data = tffl.pages[0].asarray(out='memmap')  # Use memory-mapped array
-                                tif_writer.write(data, metadata={"axes": "TYX", "fps": self.meta["frame_rate"]})
+                                data = tffl.pages[0].asarray()
+                                # Write each image directly to the output TIFF file
+                                tif_writer.write(
+                                    data[np.newaxis, ...],  # Add a new axis to match (frame x height x width)
+                                    photometric='minisblack',
+                                    metadata={'axes': 'TYX', 'fps': self.meta["frame_rate"]},
+                                    ome=True
+                                )
                 except Exception as e:
                     raise Exception(f"Error in processing tiff file {input_file}: {e}")
                 # combined_data = []

From 4b3fc89cef0edd5c951bc347cde595ea2a2f8c1f Mon Sep 17 00:00:00 2001
From: kushalbakshi <kushalbakshi@gmail.com>
Date: Thu, 20 Jun 2024 16:06:38 -0400
Subject: [PATCH 46/64] Remove `ome` kwarg

---
 element_interface/prairie_view_loader.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/element_interface/prairie_view_loader.py b/element_interface/prairie_view_loader.py
index a29f2f8..76c72bf 100644
--- a/element_interface/prairie_view_loader.py
+++ b/element_interface/prairie_view_loader.py
@@ -190,7 +190,6 @@ def write_single_bigtiff(
                                     data[np.newaxis, ...],  # Add a new axis to match (frame x height x width)
                                     photometric='minisblack',
                                     metadata={'axes': 'TYX', 'fps': self.meta["frame_rate"]},
-                                    ome=True
                                 )
                 except Exception as e:
                     raise Exception(f"Error in processing tiff file {input_file}: {e}")

From d39ca6c504de386ccf0d3f5a03b3f8eac79fb456 Mon Sep 17 00:00:00 2001
From: kushalbakshi <kushalbakshi@gmail.com>
Date: Fri, 21 Jun 2024 13:36:26 -0400
Subject: [PATCH 47/64] Add(prairie_view_loader.py) pyvips to process ome.tif
 files

---
 element_interface/prairie_view_loader.py | 44 ++++++++----------------
 1 file changed, 14 insertions(+), 30 deletions(-)

diff --git a/element_interface/prairie_view_loader.py b/element_interface/prairie_view_loader.py
index 76c72bf..50b0390 100644
--- a/element_interface/prairie_view_loader.py
+++ b/element_interface/prairie_view_loader.py
@@ -157,6 +157,9 @@ def write_single_bigtiff(
                 metadata={"axes": "TYX", "'fps'": self.meta["frame_rate"]},
                 bigtiff=True,
             )
+            combined_data = np.dstack(combined_data).transpose(
+                    2, 0, 1
+                )  # (frame x height x width)
         else:
             if not caiman_compatible:
                 with tifffile.TiffWriter(
@@ -179,39 +182,20 @@ def write_single_bigtiff(
                     except Exception as e:
                         raise Exception(f"Error in processing tiff file {input_file}: {e}")
             else:
+                import pyvips
+
+
+                combined_data = []
                 try:
-                    with tifffile.TiffWriter(output_tiff_fullpath, bigtiff=True) as tif_writer:
-                        for input_file in tiff_names:
-                            with tifffile.TiffFile(self.prairieview_dir / input_file) as tffl:
-                                assert len(tffl.pages) == 1
-                                data = tffl.pages[0].asarray()
-                                # Write each image directly to the output TIFF file
-                                tif_writer.write(
-                                    data[np.newaxis, ...],  # Add a new axis to match (frame x height x width)
-                                    photometric='minisblack',
-                                    metadata={'axes': 'TYX', 'fps': self.meta["frame_rate"]},
-                                )
+                    for input_file in tiff_names:
+                        combined_data.append(pyvips.Image.new_from_file(self.prairieview_dir / input_file, n=1))
+                    
+                    final_image = pyvips.Image.arrayjoin(combined_data, across=1)
+                    final_image.set_type(pyvips.GValue.gint_type, "page-height", self.meta["height_in_pixels"])
+                    final_image.write_to_file(output_tiff_fullpath, subif=True)
+                
                 except Exception as e:
                     raise Exception(f"Error in processing tiff file {input_file}: {e}")
-                # combined_data = []
-                # try:
-                #     for input_file in tiff_names:
-                #         with tifffile.TiffFile(self.prairieview_dir / input_file) as tffl:
-                #             assert len(tffl.pages) == 1
-                #             combined_data.append(tffl.pages[0].asarray())
-                # except Exception as e:
-                #     raise Exception(f"Error in processing tiff file {input_file}: {e}")
-
-                # combined_data = np.dstack(combined_data).transpose(
-                #     2, 0, 1
-                # )  # (frame x height x width)
-
-                # tifffile.imwrite(
-                #     output_tiff_fullpath,
-                #     combined_data,
-                #     metadata={"axes": "TYX", "'fps'": self.meta["frame_rate"]},
-                #     bigtiff=True,
-                # )
 
         return output_tiff_fullpath
 

From 2a693035325b26d3708cb6b67d90349edcdae1c7 Mon Sep 17 00:00:00 2001
From: kushalbakshi <kushalbakshi@gmail.com>
Date: Fri, 21 Jun 2024 13:50:25 -0400
Subject: [PATCH 48/64] Fix typo

---
 element_interface/prairie_view_loader.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/element_interface/prairie_view_loader.py b/element_interface/prairie_view_loader.py
index 50b0390..a5c9d07 100644
--- a/element_interface/prairie_view_loader.py
+++ b/element_interface/prairie_view_loader.py
@@ -192,7 +192,7 @@ def write_single_bigtiff(
                     
                     final_image = pyvips.Image.arrayjoin(combined_data, across=1)
                     final_image.set_type(pyvips.GValue.gint_type, "page-height", self.meta["height_in_pixels"])
-                    final_image.write_to_file(output_tiff_fullpath, subif=True)
+                    final_image.write_to_file(output_tiff_fullpath, subifd=True)
                 
                 except Exception as e:
                     raise Exception(f"Error in processing tiff file {input_file}: {e}")

From 7045aa075b7d4d926b19e4819d86628e650ce5c7 Mon Sep 17 00:00:00 2001
From: kushalbakshi <kushalbakshi@gmail.com>
Date: Fri, 21 Jun 2024 14:42:33 -0400
Subject: [PATCH 49/64] Fix(prairie_view_loader.py) set `bigtiff=True` in
 pyvips

---
 element_interface/prairie_view_loader.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/element_interface/prairie_view_loader.py b/element_interface/prairie_view_loader.py
index a5c9d07..1344ab6 100644
--- a/element_interface/prairie_view_loader.py
+++ b/element_interface/prairie_view_loader.py
@@ -192,7 +192,7 @@ def write_single_bigtiff(
                     
                     final_image = pyvips.Image.arrayjoin(combined_data, across=1)
                     final_image.set_type(pyvips.GValue.gint_type, "page-height", self.meta["height_in_pixels"])
-                    final_image.write_to_file(output_tiff_fullpath, subifd=True)
+                    final_image.write_to_file(output_tiff_fullpath, subifd=True, bigtiff=True)
                 
                 except Exception as e:
                     raise Exception(f"Error in processing tiff file {input_file}: {e}")

From 3d4d4ca1a7c4fbc5269eb820dec50031a1be6100 Mon Sep 17 00:00:00 2001
From: kushalbakshi <kushalbakshi@gmail.com>
Date: Mon, 24 Jun 2024 14:54:08 -0400
Subject: [PATCH 50/64] Fix(prairie_view_loader.py) - remove pyvips, add
 tiffwriter code from Luthi lab

---
 element_interface/prairie_view_loader.py | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/element_interface/prairie_view_loader.py b/element_interface/prairie_view_loader.py
index 1344ab6..376ff57 100644
--- a/element_interface/prairie_view_loader.py
+++ b/element_interface/prairie_view_loader.py
@@ -182,17 +182,15 @@ def write_single_bigtiff(
                     except Exception as e:
                         raise Exception(f"Error in processing tiff file {input_file}: {e}")
             else:
-                import pyvips
-
-
                 combined_data = []
                 try:
                     for input_file in tiff_names:
-                        combined_data.append(pyvips.Image.new_from_file(self.prairieview_dir / input_file, n=1))
-                    
-                    final_image = pyvips.Image.arrayjoin(combined_data, across=1)
-                    final_image.set_type(pyvips.GValue.gint_type, "page-height", self.meta["height_in_pixels"])
-                    final_image.write_to_file(output_tiff_fullpath, subifd=True, bigtiff=True)
+                        with tifffile.TiffWriter(
+                            output_tiff_fullpath,
+                            bigtiff=True,
+                        ) as tiff_writer:
+                            img = tifffile.imread(self.prairieview_dir / input_file)
+                            tiff_writer.save(img)
                 
                 except Exception as e:
                     raise Exception(f"Error in processing tiff file {input_file}: {e}")

From 188954a56a18f48acf3fae26db411bd323f193d2 Mon Sep 17 00:00:00 2001
From: kushalbakshi <kushalbakshi@gmail.com>
Date: Mon, 24 Jun 2024 15:41:50 -0400
Subject: [PATCH 51/64] Fix(prairie_view_loader.py) append to existing tiff if
 one already exists

---
 element_interface/prairie_view_loader.py | 21 +++++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)

diff --git a/element_interface/prairie_view_loader.py b/element_interface/prairie_view_loader.py
index 376ff57..b164dc5 100644
--- a/element_interface/prairie_view_loader.py
+++ b/element_interface/prairie_view_loader.py
@@ -185,12 +185,21 @@ def write_single_bigtiff(
                 combined_data = []
                 try:
                     for input_file in tiff_names:
-                        with tifffile.TiffWriter(
-                            output_tiff_fullpath,
-                            bigtiff=True,
-                        ) as tiff_writer:
-                            img = tifffile.imread(self.prairieview_dir / input_file)
-                            tiff_writer.save(img)
+                        if not output_tiff_fullpath.exists():
+                            with tifffile.TiffWriter(
+                                output_tiff_fullpath,
+                                bigtiff=True,
+                            ) as tiff_writer:
+                                img = tifffile.imread(self.prairieview_dir / input_file)
+                                tiff_writer.save(img)
+                        else:
+                            with tifffile.TiffWriter(
+                                output_tiff_fullpath,
+                                bigtiff=True,
+                                append=True,
+                            ) as tiff_writer:
+                                img = tifffile.imread(self.prairieview_dir / input_file)
+                                tiff_writer.save(img)
                 
                 except Exception as e:
                     raise Exception(f"Error in processing tiff file {input_file}: {e}")

From 990f6b16449f4df772f42e7d84d90cb8f6d6ae8d Mon Sep 17 00:00:00 2001
From: Thinh Nguyen <thinh@datajoint.com>
Date: Wed, 26 Jun 2024 11:29:54 -0500
Subject: [PATCH 52/64] fix(run_caiman): update `cnmf_output_file`

---
 element_interface/run_caiman.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/element_interface/run_caiman.py b/element_interface/run_caiman.py
index db6a059..a582a17 100644
--- a/element_interface/run_caiman.py
+++ b/element_interface/run_caiman.py
@@ -64,7 +64,7 @@ def run_caiman(
         cm.stop_server(dview=dview)
 
     cnmf_output_file = pathlib.Path(cnmf_output.mmap_file[:-4] + "hdf5")
+    cnmf_output_file = pathlib.Path(output_dir) / cnmf_output_file.name
     assert cnmf_output_file.exists()
-    assert cnmf_output_file.parent == pathlib.Path(output_dir)
 
     _save_mc(mc_output, cnmf_output_file.as_posix(), parameters["is3D"])

From dbec43425fa908df35b4381bf0d7b10036df6db4 Mon Sep 17 00:00:00 2001
From: Thinh Nguyen <thinh@datajoint.com>
Date: Wed, 26 Jun 2024 13:31:30 -0500
Subject: [PATCH 53/64] update: deprecate "caiman_compatible" argument

---
 element_interface/prairie_view_loader.py | 56 +++++++++---------------
 1 file changed, 20 insertions(+), 36 deletions(-)

diff --git a/element_interface/prairie_view_loader.py b/element_interface/prairie_view_loader.py
index f43f7ca..97a0efa 100644
--- a/element_interface/prairie_view_loader.py
+++ b/element_interface/prairie_view_loader.py
@@ -5,6 +5,9 @@
 from datetime import datetime
 import numpy as np
 import tifffile
+import logging
+
+logger = logging.getLogger(__name__)
 
 
 class PrairieViewMeta:
@@ -102,6 +105,8 @@ def write_single_bigtiff(
         caiman_compatible=False,  # if True, save the movie as a single page (frame x height x width)
         overwrite=False,
     ):
+        logger.warning("Deprecation warning: `caiman_compatible` argument will no longer have any effect and will be removed in the future. `write_single_bigtiff` will return multi-page tiff, which is compatible with CaImAn.")
+
         tiff_names, plane_idx, channel = self.get_prairieview_filenames(
             plane_idx=plane_idx, channel=channel, return_pln_chn=True
         )
@@ -109,7 +114,7 @@ def write_single_bigtiff(
             output_prefix = os.path.commonprefix(tiff_names)
         output_tiff_fullpath = (
             Path(output_dir)
-            / f"{output_prefix}_pln{plane_idx}_chn{channel}{'.ome' if not caiman_compatible else ''}.tif"
+            / f"{output_prefix}_pln{plane_idx}_chn{channel}.tif"
         )
         if output_tiff_fullpath.exists() and not overwrite:
             return output_tiff_fullpath
@@ -158,47 +163,26 @@ def write_single_bigtiff(
                 bigtiff=True,
             )
         else:
-            if not caiman_compatible:
-                with tifffile.TiffWriter(
-                    output_tiff_fullpath,
-                    bigtiff=True,
-                ) as tiff_writer:
-                    try:
-                        for input_file in tiff_names:
-                            with tifffile.TiffFile(
-                                self.prairieview_dir / input_file
-                            ) as tffl:
-                                assert len(tffl.pages) == 1
-                                tiff_writer.write(
-                                    tffl.pages[0].asarray(),
-                                    metadata={
-                                        "axes": "YX",
-                                        "'fps'": self.meta["frame_rate"],
-                                    },
-                                )
-                    except Exception as e:
-                        raise Exception(f"Error in processing tiff file {input_file}: {e}")
-            else:
-                combined_data = []
+            with tifffile.TiffWriter(
+                output_tiff_fullpath,
+                bigtiff=True,
+            ) as tiff_writer:
                 try:
                     for input_file in tiff_names:
-                        with tifffile.TiffFile(self.prairieview_dir / input_file) as tffl:
+                        with tifffile.TiffFile(
+                            self.prairieview_dir / input_file
+                        ) as tffl:
                             assert len(tffl.pages) == 1
-                            combined_data.append(tffl.pages[0].asarray())
+                            tiff_writer.write(
+                                tffl.pages[0].asarray(),
+                                metadata={
+                                    "axes": "YX",
+                                    "'fps'": self.meta["frame_rate"],
+                                },
+                            )
                 except Exception as e:
                     raise Exception(f"Error in processing tiff file {input_file}: {e}")
 
-                combined_data = np.dstack(combined_data).transpose(
-                    2, 0, 1
-                )  # (frame x height x width)
-
-                tifffile.imwrite(
-                    output_tiff_fullpath,
-                    combined_data,
-                    metadata={"axes": "TYX", "'fps'": self.meta["frame_rate"]},
-                    bigtiff=True,
-                )
-
         return output_tiff_fullpath
 
 

From 315c19af87a353b29a8bf94adf59c3b2c3185e16 Mon Sep 17 00:00:00 2001
From: Thinh Nguyen <thinh@datajoint.com>
Date: Fri, 28 Jun 2024 15:49:21 -0500
Subject: [PATCH 54/64] fix: additional safeguard to close the file and delete
 the `tffl` object

---
 element_interface/prairie_view_loader.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/element_interface/prairie_view_loader.py b/element_interface/prairie_view_loader.py
index 97a0efa..6f701ae 100644
--- a/element_interface/prairie_view_loader.py
+++ b/element_interface/prairie_view_loader.py
@@ -180,6 +180,10 @@ def write_single_bigtiff(
                                     "'fps'": self.meta["frame_rate"],
                                 },
                             )
+                        # additional safeguard to close the file and delete the object
+                        # in the attempt to prevent error: `not a TIFF file b''`
+                        tffl.close()
+                        del tffl
                 except Exception as e:
                     raise Exception(f"Error in processing tiff file {input_file}: {e}")
 

From f3aea5354df761d80f520e75b916e5c819352883 Mon Sep 17 00:00:00 2001
From: Thinh Nguyen <thinh@datajoint.com>
Date: Sat, 6 Jul 2024 08:22:44 -0500
Subject: [PATCH 55/64] feat: add `use_cuda` for caiman

---
 element_interface/run_caiman.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/element_interface/run_caiman.py b/element_interface/run_caiman.py
index d77a9e2..51fb044 100644
--- a/element_interface/run_caiman.py
+++ b/element_interface/run_caiman.py
@@ -8,6 +8,13 @@
 except:  # noqa E722
     pass  # TODO: remove bare except
 
+try:
+    import torch
+    cuda_is_available = torch.cuda.is_available()
+except:
+    cuda_is_available = False
+    pass
+
 import caiman as cm
 from caiman.source_extraction.cnmf import params as params
 from caiman.source_extraction.cnmf.cnmf import CNMF
@@ -36,6 +43,8 @@ def run_caiman(
     parameters["fnames"] = file_paths
     parameters["fr"] = sampling_rate
 
+    parameters["use_cuda"] = cuda_is_available  # Use CUDA if available
+
     if "indices" in parameters:
         indices = parameters.pop(
             "indices"

From b525c8c6cded506c0a67663d0577dfa36dc0d307 Mon Sep 17 00:00:00 2001
From: Thinh Nguyen <thinh@datajoint.com>
Date: Tue, 23 Jul 2024 09:20:30 -0500
Subject: [PATCH 56/64] feat: set `gb_per_file` and write multiple bigtiff

---
 element_interface/caiman_loader.py       | 77 +++++++++++++----------
 element_interface/prairie_view_loader.py | 78 ++++++++++++++----------
 2 files changed, 91 insertions(+), 64 deletions(-)

diff --git a/element_interface/caiman_loader.py b/element_interface/caiman_loader.py
index e245abd..18199c2 100644
--- a/element_interface/caiman_loader.py
+++ b/element_interface/caiman_loader.py
@@ -532,7 +532,12 @@ def _process_scanimage_tiff(scan_filenames, output_dir="./", split_depths=False)
             imsave(save_fp.as_posix(), chn_vol)
 
 
-def _save_mc(mc, caiman_fp: str, is3D: bool):
+def _save_mc(
+    mc,
+    caiman_fp: str,
+    is3D: bool,
+    summary_images: dict = None,
+):
     """Save motion correction to hdf5 output
 
     Run these commands after the CaImAn analysis has completed.
@@ -545,21 +550,13 @@ def _save_mc(mc, caiman_fp: str, is3D: bool):
             shifts_rig :   Rigid transformation x and y shifts per frame
             x_shifts_els : Non rigid transformation x shifts per frame per block
             y_shifts_els : Non rigid transformation y shifts per frame per block
-        caiman_fp (str): CaImAn output (*.hdf5) file path
+        caiman_fp (str): CaImAn output (*.hdf5) file path - append if exists, else create new one
+        is3D (bool):  the data is 3D
+        summary_images(dict): dict of summary images (average_image, max_image, correlation_image) - if None, will be computed, if provided as empty dict, will not be computed
     """
-
-    # Load motion corrected mmap image
-    mc_image = cm.load(mc.mmap_file, is3D=is3D)
-
-    # Compute motion corrected summary images
-    average_image = np.mean(mc_image, axis=0)
-    max_image = np.max(mc_image, axis=0)
-
-    # Compute motion corrected correlation image
-    correlation_image = cm.local_correlations(
-        mc_image.transpose((1, 2, 3, 0) if is3D else (1, 2, 0))
-    )
-    correlation_image[np.isnan(correlation_image)] = 0
+    Yr, dims, T = cm.mmapping.load_memmap(mc.mmap_file[0])
+    # Load the first frame of the movie
+    mc_image = np.reshape(Yr[: np.product(dims), :1], [1] + list(dims), order="F")
 
     # Compute mc.coord_shifts_els
     grid = []
@@ -591,7 +588,8 @@ def _save_mc(mc, caiman_fp: str, is3D: bool):
             )
 
     # Open hdf5 file and create 'motion_correction' group
-    h5f = h5py.File(caiman_fp, "r+")
+    caiman_fp = pathlib.Path(caiman_fp)
+    h5f = h5py.File(caiman_fp, "r+" if caiman_fp.exists() else "w")
     h5g = h5f.require_group("motion_correction")
 
     # Write motion correction shifts and motion corrected summary images to hdf5 file
@@ -623,7 +621,7 @@ def _save_mc(mc, caiman_fp: str, is3D: bool):
         # For CaImAn, reference image is still a 2D array even for the case of 3D
         # Assume that the same ref image is used for all the planes
         reference_image = (
-            np.tile(mc.total_template_els, (1, 1, correlation_image.shape[-1]))
+            np.tile(mc.total_template_els, (1, 1, dims[-1]))
             if is3D
             else mc.total_template_els
         )
@@ -638,32 +636,45 @@ def _save_mc(mc, caiman_fp: str, is3D: bool):
             "coord_shifts_rig", shape=np.shape(grid), data=grid, dtype=type(grid[0][0])
         )
         reference_image = (
-            np.tile(mc.total_template_rig, (1, 1, correlation_image.shape[-1]))
+            np.tile(mc.total_template_rig, (1, 1, dims[-1]))
             if is3D
             else mc.total_template_rig
         )
 
+    if summary_images is None:
+        # Load motion corrected mmap image
+        mc_image = cm.load(mc.mmap_file, is3D=is3D)
+
+        # Compute motion corrected summary images
+        average_image = np.mean(mc_image, axis=0)
+        max_image = np.max(mc_image, axis=0)
+
+        # Compute motion corrected correlation image
+        correlation_image = cm.local_correlations(
+            mc_image.transpose((1, 2, 3, 0) if is3D else (1, 2, 0))
+        )
+        correlation_image[np.isnan(correlation_image)] = 0
+
+        summary_images = {
+            "average_image": average_image,
+            "max_image": max_image,
+            "correlation_image": correlation_image,
+        }
+
+    for img_type, img in summary_images.items():
+        h5g.require_dataset(
+            img_type,
+            shape=np.shape(img),
+            data=img,
+            dtype=img.dtype,
+        )
+
     h5g.require_dataset(
         "reference_image",
         shape=np.shape(reference_image),
         data=reference_image,
         dtype=reference_image.dtype,
     )
-    h5g.require_dataset(
-        "correlation_image",
-        shape=np.shape(correlation_image),
-        data=correlation_image,
-        dtype=correlation_image.dtype,
-    )
-    h5g.require_dataset(
-        "average_image",
-        shape=np.shape(average_image),
-        data=average_image,
-        dtype=average_image.dtype,
-    )
-    h5g.require_dataset(
-        "max_image", shape=np.shape(max_image), data=max_image, dtype=max_image.dtype
-    )
 
     # Close hdf5 file
     h5f.close()
diff --git a/element_interface/prairie_view_loader.py b/element_interface/prairie_view_loader.py
index 6f701ae..83d56b2 100644
--- a/element_interface/prairie_view_loader.py
+++ b/element_interface/prairie_view_loader.py
@@ -104,6 +104,7 @@ def write_single_bigtiff(
         output_dir="./",
         caiman_compatible=False,  # if True, save the movie as a single page (frame x height x width)
         overwrite=False,
+        gb_per_file=None,
     ):
         logger.warning("Deprecation warning: `caiman_compatible` argument will no longer have any effect and will be removed in the future. `write_single_bigtiff` will return multi-page tiff, which is compatible with CaImAn.")
 
@@ -112,13 +113,14 @@ def write_single_bigtiff(
         )
         if output_prefix is None:
             output_prefix = os.path.commonprefix(tiff_names)
-        output_tiff_fullpath = (
-            Path(output_dir)
-            / f"{output_prefix}_pln{plane_idx}_chn{channel}.tif"
-        )
-        if output_tiff_fullpath.exists() and not overwrite:
-            return output_tiff_fullpath
+        output_tiff_stem = f"{output_prefix}_pln{plane_idx}_chn{channel}"
+
+        output_dir = Path(output_dir)
+        output_tiff_list = list(output_dir.glob(f"{output_tiff_stem}*.tif"))
+        if len(output_tiff_list) and not overwrite:
+            return output_tiff_list[0] if gb_per_file is None else output_tiff_list
 
+        output_tiff_list = []
         if self.meta["is_multipage"]:
             # For multi-page tiff - the pages are organized as:
             # (channel x slice x frame) - each page is (height x width)
@@ -156,38 +158,52 @@ def write_single_bigtiff(
             except Exception as e:
                 raise Exception(f"Error in processing tiff file {input_file}: {e}")
 
+            output_tiff_fullpath = (
+                    output_dir
+                    / f"{output_tiff_stem}.tif"
+            )
             tifffile.imwrite(
                 output_tiff_fullpath,
                 combined_data,
                 metadata={"axes": "TYX", "'fps'": self.meta["frame_rate"]},
                 bigtiff=True,
             )
+            output_tiff_list.append(output_tiff_fullpath)
         else:
-            with tifffile.TiffWriter(
-                output_tiff_fullpath,
-                bigtiff=True,
-            ) as tiff_writer:
-                try:
-                    for input_file in tiff_names:
-                        with tifffile.TiffFile(
-                            self.prairieview_dir / input_file
-                        ) as tffl:
-                            assert len(tffl.pages) == 1
-                            tiff_writer.write(
-                                tffl.pages[0].asarray(),
-                                metadata={
-                                    "axes": "YX",
-                                    "'fps'": self.meta["frame_rate"],
-                                },
-                            )
-                        # additional safeguard to close the file and delete the object
-                        # in the attempt to prevent error: `not a TIFF file b''`
-                        tffl.close()
-                        del tffl
-                except Exception as e:
-                    raise Exception(f"Error in processing tiff file {input_file}: {e}")
-
-        return output_tiff_fullpath
+            while len(tiff_names):
+                output_tiff_fullpath = (
+                        output_dir
+                        / f"{output_tiff_stem}_{len(output_tiff_list):04}.tif"
+                )
+                with tifffile.TiffWriter(
+                    output_tiff_fullpath,
+                    bigtiff=True,
+                ) as tiff_writer:
+                    while len(tiff_names):
+                        input_file = tiff_names.pop(0)
+                        try:
+                            with tifffile.TiffFile(
+                                self.prairieview_dir / input_file
+                            ) as tffl:
+                                assert len(tffl.pages) == 1
+                                tiff_writer.write(
+                                    tffl.pages[0].asarray(),
+                                    metadata={
+                                        "axes": "YX",
+                                        "'fps'": self.meta["frame_rate"],
+                                    },
+                                )
+                            # additional safeguard to close the file and delete the object
+                            # in the attempt to prevent error: `not a TIFF file b''`
+                            tffl.close()
+                            del tffl
+                        except Exception as e:
+                            raise Exception(f"Error in processing tiff file {input_file}: {e}")
+                        if gb_per_file and output_tiff_fullpath.stat().st_size >= gb_per_file * 1024 ** 3:
+                            break
+                    output_tiff_list.append(output_tiff_fullpath)
+
+        return output_tiff_list[0] if gb_per_file is None else output_tiff_list
 
 
 def _extract_prairieview_metadata(xml_filepath: str):

From dc98ebc4e71f4dc62b0a902ee5cb29df1e4434a0 Mon Sep 17 00:00:00 2001
From: Thinh Nguyen <thinh@datajoint.com>
Date: Tue, 23 Jul 2024 14:55:45 -0500
Subject: [PATCH 57/64] chore: add warning with `gb_per_file` for
 `is_multipage` tiff

---
 element_interface/prairie_view_loader.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/element_interface/prairie_view_loader.py b/element_interface/prairie_view_loader.py
index 83d56b2..13b8c89 100644
--- a/element_interface/prairie_view_loader.py
+++ b/element_interface/prairie_view_loader.py
@@ -122,6 +122,8 @@ def write_single_bigtiff(
 
         output_tiff_list = []
         if self.meta["is_multipage"]:
+            if gb_per_file is not None:
+                logger.warning("Ignoring `gb_per_file` argument for multi-page tiff (NotYetImplemented)")
             # For multi-page tiff - the pages are organized as:
             # (channel x slice x frame) - each page is (height x width)
             # - TODO: verify this is the case for Bruker multi-page tiff

From e38352e97283eeeb3c7f105bb85e42c197e76a9a Mon Sep 17 00:00:00 2001
From: Thinh Nguyen <thinh@datajoint.com>
Date: Wed, 24 Jul 2024 13:04:24 -0500
Subject: [PATCH 58/64] fix: remove old tif files if `overwrite=True`

---
 element_interface/prairie_view_loader.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/element_interface/prairie_view_loader.py b/element_interface/prairie_view_loader.py
index 13b8c89..a3e9790 100644
--- a/element_interface/prairie_view_loader.py
+++ b/element_interface/prairie_view_loader.py
@@ -120,6 +120,9 @@ def write_single_bigtiff(
         if len(output_tiff_list) and not overwrite:
             return output_tiff_list[0] if gb_per_file is None else output_tiff_list
 
+        # delete old tif files if overwrite is True
+        [f.unlink() for f in output_tiff_list]
+
         output_tiff_list = []
         if self.meta["is_multipage"]:
             if gb_per_file is not None:

From 0b3b306ca623626412d22e8f5a0252c38480b752 Mon Sep 17 00:00:00 2001
From: Thinh Nguyen <thinh@datajoint.com>
Date: Wed, 24 Jul 2024 13:47:13 -0500
Subject: [PATCH 59/64] fix: incorrect dimension ordering for single-plane
 caimain loading

---
 element_interface/caiman_loader.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/element_interface/caiman_loader.py b/element_interface/caiman_loader.py
index 18199c2..7d1e06f 100644
--- a/element_interface/caiman_loader.py
+++ b/element_interface/caiman_loader.py
@@ -289,7 +289,7 @@ def _get_image(self, img_type):
             img_ = (
                 pln_cm.motion_correction[img_type].transpose()
                 if self.is3D
-                else pln_cm.motion_correction[img_type][...][np.newaxis, ...]
+                else pln_cm.motion_correction[img_type][...][..., np.newaxis]
             )
         else:
             img_ = np.dstack(

From eb5254dea2fbd1a9f1df259d169713c1cb4e0ff8 Mon Sep 17 00:00:00 2001
From: Thinh Nguyen <thinh@datajoint.com>
Date: Tue, 6 Aug 2024 13:17:18 -0500
Subject: [PATCH 60/64] feat(run_caiman): use `output_dir` for `CAIMAN_TEMP`

---
 element_interface/run_caiman.py | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/element_interface/run_caiman.py b/element_interface/run_caiman.py
index 51fb044..7c9e514 100644
--- a/element_interface/run_caiman.py
+++ b/element_interface/run_caiman.py
@@ -2,6 +2,8 @@
 import os
 import pathlib
 import shutil
+import numpy as np
+import multiprocessing
 
 try:
     cv2.setNumThreads(0)
@@ -43,7 +45,8 @@ def run_caiman(
     parameters["fnames"] = file_paths
     parameters["fr"] = sampling_rate
 
-    parameters["use_cuda"] = cuda_is_available  # Use CUDA if available
+    use_cuda = parameters.get("use_cuda")
+    parameters["use_cuda"] = cuda_is_available if use_cuda is None else use_cuda
 
     if "indices" in parameters:
         indices = parameters.pop(
@@ -52,13 +55,17 @@ def run_caiman(
         indices = slice(*indices[0]), slice(*indices[1])
         parameters["motion"] = {**parameters.get("motion", {}), "indices": indices}
 
-    opts = params.CNMFParams(params_dict=parameters)
+    caiman_temp = os.environ.get("CAIMAN_TEMP")
+    os.environ["CAIMAN_TEMP"] = str(output_dir)
 
-    c, dview, n_processes = cm.cluster.setup_cluster(
-        backend="multiprocessing", n_processes=None
+    # use 80% of available cores
+    n_processes = int(np.floor(multiprocessing.cpu_count() * 0.8))
+    _, dview, n_processes = cm.cluster.setup_cluster(
+        backend="multiprocessing", n_processes=n_processes
     )
 
     try:
+        opts = params.CNMFParams(params_dict=parameters)
         cnm = CNMF(n_processes, params=opts, dview=dview)
         cnmf_output, mc_output = cnm.fit_file(
             motion_correct=True,
@@ -73,6 +80,11 @@ def run_caiman(
     else:
         cm.stop_server(dview=dview)
 
+    if caiman_temp is not None:
+        os.environ["CAIMAN_TEMP"] = caiman_temp
+    else:
+        del os.environ["CAIMAN_TEMP"]
+
     cnmf_output_file = pathlib.Path(cnmf_output.mmap_file[:-4] + "hdf5")
     cnmf_output_file = pathlib.Path(output_dir) / cnmf_output_file.name
     assert cnmf_output_file.exists()

From 686bed62e38da375181e1c191d8596a493db9ae5 Mon Sep 17 00:00:00 2001
From: kushalbakshi <kushalbakshi@gmail.com>
Date: Fri, 9 Aug 2024 15:34:14 -0400
Subject: [PATCH 61/64] Update docs, GHA, version, CHANGELOG

---
 .github/workflows/mkdocs-release-caller.yml   |  9 +++++
 .github/workflows/release.yaml                | 10 ++++++
 .github/workflows/semantic-release-caller.yml |  7 ++++
 .github/workflows/test.yaml                   | 36 +++++++++++++++++++
 .../workflows/u24_element_before_release.yaml | 17 ---------
 .../workflows/u24_element_release_call.yaml   | 28 ---------------
 .../workflows/u24_element_tag_to_release.yaml | 14 --------
 CHANGELOG.md                                  |  8 ++++-
 LICENSE                                       |  2 +-
 docs/src/concepts.md                          | 22 ++++++++----
 docs/src/index.md                             |  2 ++
 element_interface/version.py                  |  2 +-
 12 files changed, 88 insertions(+), 69 deletions(-)
 create mode 100644 .github/workflows/mkdocs-release-caller.yml
 create mode 100644 .github/workflows/release.yaml
 create mode 100644 .github/workflows/semantic-release-caller.yml
 create mode 100644 .github/workflows/test.yaml
 delete mode 100644 .github/workflows/u24_element_before_release.yaml
 delete mode 100644 .github/workflows/u24_element_release_call.yaml
 delete mode 100644 .github/workflows/u24_element_tag_to_release.yaml

diff --git a/.github/workflows/mkdocs-release-caller.yml b/.github/workflows/mkdocs-release-caller.yml
new file mode 100644
index 0000000..14e6f4f
--- /dev/null
+++ b/.github/workflows/mkdocs-release-caller.yml
@@ -0,0 +1,9 @@
+name: mkdocs-release
+on:
+  workflow_dispatch:
+
+jobs:
+  mkdocs_release:
+    uses: datajoint/.github/.github/workflows/mkdocs_release.yaml@main
+    permissions: 
+      contents: write
\ No newline at end of file
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
new file mode 100644
index 0000000..954c84a
--- /dev/null
+++ b/.github/workflows/release.yaml
@@ -0,0 +1,10 @@
+name: Release
+on:
+  workflow_dispatch:
+jobs:
+  make_github_release:
+    uses: datajoint/.github/.github/workflows/make_github_release.yaml@main
+  mkdocs_release:
+    uses: datajoint/.github/.github/workflows/mkdocs_release.yaml@main
+    permissions: 
+      contents: write
\ No newline at end of file
diff --git a/.github/workflows/semantic-release-caller.yml b/.github/workflows/semantic-release-caller.yml
new file mode 100644
index 0000000..2aa3cd1
--- /dev/null
+++ b/.github/workflows/semantic-release-caller.yml
@@ -0,0 +1,7 @@
+name: semantic-release
+on:
+  workflow_dispatch:
+
+jobs:
+  call_semantic_release:
+    uses: datajoint/.github/.github/workflows/semantic-release.yaml@main
\ No newline at end of file
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
new file mode 100644
index 0000000..b8a27c1
--- /dev/null
+++ b/.github/workflows/test.yaml
@@ -0,0 +1,36 @@
+name: Test
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 8 * * 1"
+jobs:
+  devcontainer-build:
+    uses: datajoint/.github/.github/workflows/devcontainer-build.yaml@main
+  tests:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        py_ver: ["3.9", "3.10"]
+        mysql_ver: ["8.0", "5.7"]
+        include:
+          - py_ver: "3.8"
+            mysql_ver: "5.7"
+          - py_ver: "3.7"
+            mysql_ver: "5.7"
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{matrix.py_ver}}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{matrix.py_ver}}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install flake8 "black[jupyter]"
+      - name: Run style tests
+        run: |
+          python_version=${{matrix.py_ver}}
+          black element_interface --check --verbose --target-version py${python_version//.}
+          black notebooks --check --verbose --target-version py${python_version//.}
\ No newline at end of file
diff --git a/.github/workflows/u24_element_before_release.yaml b/.github/workflows/u24_element_before_release.yaml
deleted file mode 100644
index 692cf82..0000000
--- a/.github/workflows/u24_element_before_release.yaml
+++ /dev/null
@@ -1,17 +0,0 @@
-name: u24_element_before_release
-on:
-  pull_request:
-  push:
-    branches:
-      - '**'
-    tags-ignore:
-      - '**'
-  workflow_dispatch:
-jobs:
-  call_context_check:
-    uses: dj-sciops/djsciops-cicd/.github/workflows/context_check.yaml@main
-  call_u24_elements_build_alpine:
-    uses: dj-sciops/djsciops-cicd/.github/workflows/u24_element_build.yaml@main
-    with:
-      py_ver: 3.9
-      image: djbase
diff --git a/.github/workflows/u24_element_release_call.yaml b/.github/workflows/u24_element_release_call.yaml
deleted file mode 100644
index 4324cca..0000000
--- a/.github/workflows/u24_element_release_call.yaml
+++ /dev/null
@@ -1,28 +0,0 @@
-name: u24_element_release_call
-on:
-  workflow_run:
-    workflows: ["u24_element_tag_to_release"]
-    types:
-      - completed
-jobs:
-  call_context_check:
-    uses: dj-sciops/djsciops-cicd/.github/workflows/context_check.yaml@main
-  test_call_u24_elements_release_alpine:
-    if: >-
-      github.event.workflow_run.conclusion == 'success' && ( contains(github.event.workflow_run.head_branch, 'test') || (github.event.workflow_run.event == 'pull_request'))
-    uses: dj-sciops/djsciops-cicd/.github/workflows/u24_element_release.yaml@main
-    with:
-      py_ver: 3.9
-      twine_repo: testpypi
-    secrets:
-      TWINE_USERNAME: ${{secrets.TWINE_TEST_USERNAME}}
-      TWINE_PASSWORD: ${{secrets.TWINE_TEST_PASSWORD}}
-  call_u24_elements_release_alpine:
-    if: >-
-      github.event.workflow_run.conclusion == 'success' && github.repository_owner == 'datajoint' && !contains(github.event.workflow_run.head_branch, 'test')
-    uses: dj-sciops/djsciops-cicd/.github/workflows/u24_element_release.yaml@main
-    with:
-      py_ver: 3.9
-    secrets:
-      TWINE_USERNAME: ${{secrets.TWINE_USERNAME}}
-      TWINE_PASSWORD: ${{secrets.TWINE_PASSWORD}}
diff --git a/.github/workflows/u24_element_tag_to_release.yaml b/.github/workflows/u24_element_tag_to_release.yaml
deleted file mode 100644
index 57334e9..0000000
--- a/.github/workflows/u24_element_tag_to_release.yaml
+++ /dev/null
@@ -1,14 +0,0 @@
-name: u24_element_tag_to_release
-on:
-  push:
-    tags:
-      - '*.*.*'
-      - 'test*.*.*'
-jobs:
-  call_context_check:
-    uses: dj-sciops/djsciops-cicd/.github/workflows/context_check.yaml@main
-  call_u24_elements_build_alpine:
-    uses: dj-sciops/djsciops-cicd/.github/workflows/u24_element_build.yaml@main
-    with:
-      py_ver: 3.9
-      image: djbase
diff --git a/CHANGELOG.md b/CHANGELOG.md
index f4256fb..403d3cc 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,6 +3,12 @@
 Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and
 [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) convention.
 
+## [0.8.0] - 2024-08-09
+
++ Add - `memoized_result` decorator to cache function results
++ Update - `prairie_view_loader.py` to create big tiff files from `.ome.tif` files
++ Update - `run_caiman.py` to run latest version of CaImAn
++ Update - `caiman_loader.py` to process output of latest version of CaImAn
 
 ## [0.7.0] - 2023-09-05
 
@@ -14,7 +20,6 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and
 
 + Update DANDI upload funtionality to improve useability
 
-
 ## [0.6.0] - 2023-07-26
 
 + Update - `prairieviewreader.py` -> `prairie_view_loader.py`
@@ -89,6 +94,7 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and
 
 + Add - Readers for: `ScanImage`, `Suite2p`, `CaImAn`.
 
+[0.8.0]: https://github.com/datajoint/element-interface/releases/tag/0.8.0
 [0.7.0]: https://github.com/datajoint/element-interface/releases/tag/0.7.0
 [0.6.0]: https://github.com/datajoint/element-interface/releases/tag/0.6.0
 [0.5.4]: https://github.com/datajoint/element-interface/releases/tag/0.5.4
diff --git a/LICENSE b/LICENSE
index d394fe3..6872305 100644
--- a/LICENSE
+++ b/LICENSE
@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2021 DataJoint NEURO
+Copyright (c) 2024 DataJoint
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
diff --git a/docs/src/concepts.md b/docs/src/concepts.md
index c7a592c..0ae80c2 100644
--- a/docs/src/concepts.md
+++ b/docs/src/concepts.md
@@ -24,6 +24,12 @@ how to use various Elements.
 `utils.str_to_bool` converts a set of strings to boolean True or False. This is implemented
 as the equivalent item in Python's `distutils` which will be removed in future versions.
 
+`utils.memoized_result` is a decorator that caches the result of a function call based 
+ on input parameters and the state of the output. If the function is called with the same
+ parameters and the output files in the directory remain unchanged, it returns the 
+ cached results; otherwise, it executes the function and caches the new results along 
+ with metadata.
+
 ### Suite2p
 
 This Element provides functions to independently run Suite2p's motion correction,
@@ -46,13 +52,15 @@ Requirements:
 
 ### PrairieView Reader
 
-This Element provides a function to read the PrairieView Scanner's metadata file. The
-PrairieView software generates one `.ome.tif` imaging file per frame acquired. The
-metadata for all frames is contained in one `.xml` file. This function locates the
-`.xml` file and generates a dictionary necessary to populate the DataJoint ScanInfo and
-Field tables. PrairieView works with resonance scanners with a single field, does not
-support bidirectional x and y scanning, and the `.xml` file does not contain ROI
-information.
+This Element provides a `PrairieViewMeta` class to handle different types of output from
+ the PrairieView Scanner. The PrairieView software either generates one `.ome.tif` 
+ imaging file per frame acquired or multi-page `.ome.tif` files. The metadata for all 
+ frames is contained in one `.xml` file. This class contains methods that locate the 
+ `.xml` file and generate a dictionary necessary to populate the DataJoint ScanInfo and
+  Field tables. The class also contains methods to create a big tiff file from the 
+  individual `.ome.tif` files. PrairieView works with resonance scanners with a single 
+  field, does not support bidirectional x and y scanning, and the `.xml` file does not 
+  contain ROI information.
 
 ## Element Architecture
 
diff --git a/docs/src/index.md b/docs/src/index.md
index c3eea7b..52b39f9 100644
--- a/docs/src/index.md
+++ b/docs/src/index.md
@@ -16,4 +16,6 @@ a number of other Elements.
 
 - Data ingestion, see [`ingest_csv_to_table` API](./api/element_interface/utils/#element_interface.utils.ingest_csv_to_table)
 
+- Code execution, see [`memoized_result` API](./api/element_interface/utils/#element_interface.utils.memoized_result)
+
 Visit the [Concepts page](./concepts.md) for more information on these tools.
diff --git a/element_interface/version.py b/element_interface/version.py
index 70aab85..42c688b 100644
--- a/element_interface/version.py
+++ b/element_interface/version.py
@@ -1,3 +1,3 @@
 """Package metadata"""
 
-__version__ = "0.7.0"
+__version__ = "0.8.0"

From bcd6386cc9c83c10744f2b9e8695365df20bef87 Mon Sep 17 00:00:00 2001
From: kushalbakshi <kushalbakshi@gmail.com>
Date: Fri, 9 Aug 2024 15:36:40 -0400
Subject: [PATCH 62/64] Apply black formatting

---
 element_interface/extract_trigger.py     | 17 ++++---
 element_interface/prairie_view_loader.py | 56 ++++++++++++++++--------
 element_interface/run_caiman.py          |  1 +
 element_interface/suite2p_loader.py      |  4 +-
 4 files changed, 52 insertions(+), 26 deletions(-)

diff --git a/element_interface/extract_trigger.py b/element_interface/extract_trigger.py
index 103e3e0..2600a93 100644
--- a/element_interface/extract_trigger.py
+++ b/element_interface/extract_trigger.py
@@ -43,7 +43,8 @@ def __init__(
     def write_matlab_run_script(self):
         """Compose a matlab script and save it with the name run_extract.m.
 
-        The composed script is basically the formatted version of the m_template attribute."""
+        The composed script is basically the formatted version of the m_template attribute.
+        """
 
         self.output_fullpath = (
             self.output_dir / f"{self.scanfile.stem}_extract_output.mat"
@@ -53,11 +54,15 @@ def write_matlab_run_script(self):
             **dict(
                 parameters_list_string="\n".join(
                     [
-                        f"config.{k} = '{v}';"
-                        if isinstance(v, str)
-                        else f"config.{k} = {str(v).lower()};"
-                        if isinstance(v, bool)
-                        else f"config.{k} = {v};"
+                        (
+                            f"config.{k} = '{v}';"
+                            if isinstance(v, str)
+                            else (
+                                f"config.{k} = {str(v).lower()};"
+                                if isinstance(v, bool)
+                                else f"config.{k} = {v};"
+                            )
+                        )
                         for k, v in self.parameters.items()
                     ]
                 ),
diff --git a/element_interface/prairie_view_loader.py b/element_interface/prairie_view_loader.py
index a3e9790..33fe1a9 100644
--- a/element_interface/prairie_view_loader.py
+++ b/element_interface/prairie_view_loader.py
@@ -106,7 +106,9 @@ def write_single_bigtiff(
         overwrite=False,
         gb_per_file=None,
     ):
-        logger.warning("Deprecation warning: `caiman_compatible` argument will no longer have any effect and will be removed in the future. `write_single_bigtiff` will return multi-page tiff, which is compatible with CaImAn.")
+        logger.warning(
+            "Deprecation warning: `caiman_compatible` argument will no longer have any effect and will be removed in the future. `write_single_bigtiff` will return multi-page tiff, which is compatible with CaImAn."
+        )
 
         tiff_names, plane_idx, channel = self.get_prairieview_filenames(
             plane_idx=plane_idx, channel=channel, return_pln_chn=True
@@ -126,7 +128,9 @@ def write_single_bigtiff(
         output_tiff_list = []
         if self.meta["is_multipage"]:
             if gb_per_file is not None:
-                logger.warning("Ignoring `gb_per_file` argument for multi-page tiff (NotYetImplemented)")
+                logger.warning(
+                    "Ignoring `gb_per_file` argument for multi-page tiff (NotYetImplemented)"
+                )
             # For multi-page tiff - the pages are organized as:
             # (channel x slice x frame) - each page is (height x width)
             # - TODO: verify this is the case for Bruker multi-page tiff
@@ -137,36 +141,45 @@ def write_single_bigtiff(
             slice_idx = self.meta["plane_indices"].index(plane_idx)
             channel_idx = self.meta["channels"].index(channel)
 
-            page_indices = [frame_idx * frame_step + slice_idx * slice_step + channel_idx
-                            for frame_idx in range(self.meta["num_frames"])]
+            page_indices = [
+                frame_idx * frame_step + slice_idx * slice_step + channel_idx
+                for frame_idx in range(self.meta["num_frames"])
+            ]
 
-            combined_data = np.empty([self.meta["num_frames"],
-                                      self.meta["height_in_pixels"],
-                                      self.meta["width_in_pixels"]],
-                                     dtype=int)
+            combined_data = np.empty(
+                [
+                    self.meta["num_frames"],
+                    self.meta["height_in_pixels"],
+                    self.meta["width_in_pixels"],
+                ],
+                dtype=int,
+            )
             start_page = 0
             try:
                 for input_file in tiff_names:
                     with tifffile.TiffFile(self.prairieview_dir / input_file) as tffl:
                         # Get indices in this tiff file and in output array
                         final_page_in_file = start_page + len(tffl.pages)
-                        is_page_in_file = lambda page: page in range(start_page, final_page_in_file)
+                        is_page_in_file = lambda page: page in range(
+                            start_page, final_page_in_file
+                        )
                         pages_in_file = filter(is_page_in_file, page_indices)
                         file_indices = [page - start_page for page in pages_in_file]
-                        global_indices = [is_page_in_file(page) for page in page_indices]
+                        global_indices = [
+                            is_page_in_file(page) for page in page_indices
+                        ]
 
                         # Read from this tiff file (if needed)
                         if len(file_indices) > 0:
                             # this line looks a bit ugly but is memory efficient. Do not separate
-                            combined_data[global_indices] = tffl.asarray(key=file_indices)
+                            combined_data[global_indices] = tffl.asarray(
+                                key=file_indices
+                            )
                         start_page += len(tffl.pages)
             except Exception as e:
                 raise Exception(f"Error in processing tiff file {input_file}: {e}")
 
-            output_tiff_fullpath = (
-                    output_dir
-                    / f"{output_tiff_stem}.tif"
-            )
+            output_tiff_fullpath = output_dir / f"{output_tiff_stem}.tif"
             tifffile.imwrite(
                 output_tiff_fullpath,
                 combined_data,
@@ -177,8 +190,7 @@ def write_single_bigtiff(
         else:
             while len(tiff_names):
                 output_tiff_fullpath = (
-                        output_dir
-                        / f"{output_tiff_stem}_{len(output_tiff_list):04}.tif"
+                    output_dir / f"{output_tiff_stem}_{len(output_tiff_list):04}.tif"
                 )
                 with tifffile.TiffWriter(
                     output_tiff_fullpath,
@@ -203,8 +215,14 @@ def write_single_bigtiff(
                             tffl.close()
                             del tffl
                         except Exception as e:
-                            raise Exception(f"Error in processing tiff file {input_file}: {e}")
-                        if gb_per_file and output_tiff_fullpath.stat().st_size >= gb_per_file * 1024 ** 3:
+                            raise Exception(
+                                f"Error in processing tiff file {input_file}: {e}"
+                            )
+                        if (
+                            gb_per_file
+                            and output_tiff_fullpath.stat().st_size
+                            >= gb_per_file * 1024**3
+                        ):
                             break
                     output_tiff_list.append(output_tiff_fullpath)
 
diff --git a/element_interface/run_caiman.py b/element_interface/run_caiman.py
index 7c9e514..4eefadc 100644
--- a/element_interface/run_caiman.py
+++ b/element_interface/run_caiman.py
@@ -12,6 +12,7 @@
 
 try:
     import torch
+
     cuda_is_available = torch.cuda.is_available()
 except:
     cuda_is_available = False
diff --git a/element_interface/suite2p_loader.py b/element_interface/suite2p_loader.py
index e16fd3f..646c01b 100644
--- a/element_interface/suite2p_loader.py
+++ b/element_interface/suite2p_loader.py
@@ -153,7 +153,9 @@ def __init__(self, suite2p_plane_dir: str):
 
     @property
     def curation_time(self):
-        print("DeprecationWarning: 'curation_time' is deprecated, set to be the same as 'creation time', no longer reliable.")
+        print(
+            "DeprecationWarning: 'curation_time' is deprecated, set to be the same as 'creation time', no longer reliable."
+        )
         return self.creation_time
 
     @property

From 76c27b021d8efd9da58b35195be4fdfdda26ac66 Mon Sep 17 00:00:00 2001
From: kushalbakshi <kushalbakshi@gmail.com>
Date: Fri, 9 Aug 2024 15:41:43 -0400
Subject: [PATCH 63/64] Remove notebooks from GHA checks

---
 .github/workflows/test.yaml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
index b8a27c1..bd23395 100644
--- a/.github/workflows/test.yaml
+++ b/.github/workflows/test.yaml
@@ -32,5 +32,4 @@ jobs:
       - name: Run style tests
         run: |
           python_version=${{matrix.py_ver}}
-          black element_interface --check --verbose --target-version py${python_version//.}
-          black notebooks --check --verbose --target-version py${python_version//.}
\ No newline at end of file
+          black element_interface --check --verbose --target-version py${python_version//.}
\ No newline at end of file

From d7b62113c8237ecfe1e2e7bd4e8461b9507311bd Mon Sep 17 00:00:00 2001
From: kushalbakshi <kushalbakshi@gmail.com>
Date: Fri, 9 Aug 2024 15:45:28 -0400
Subject: [PATCH 64/64] Fix version + CHANGELOG

---
 CHANGELOG.md                 | 11 +++--------
 element_interface/version.py |  2 +-
 2 files changed, 4 insertions(+), 9 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 403d3cc..815952c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,18 +3,13 @@
 Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and
 [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) convention.
 
-## [0.8.0] - 2024-08-09
+## [0.7.0] - 2024-08-09
 
 + Add - `memoized_result` decorator to cache function results
 + Update - `prairie_view_loader.py` to create big tiff files from `.ome.tif` files
 + Update - `run_caiman.py` to run latest version of CaImAn
 + Update - `caiman_loader.py` to process output of latest version of CaImAn
-
-## [0.7.0] - 2023-09-05
-
-+ Add - multi-plane `caiman_loader.py` to process multi-plane tiffs
-+ Update - DANDI upload utility
-+ Fix - `n_fields` == 1 -> `n_fields` == `n_depths` in `prairie_view_loader.py`
++ Fix - general fixes and improvements
 
 ## [0.6.1] - 2023-08-02
 
@@ -94,7 +89,7 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and
 
 + Add - Readers for: `ScanImage`, `Suite2p`, `CaImAn`.
 
-[0.8.0]: https://github.com/datajoint/element-interface/releases/tag/0.8.0
+
 [0.7.0]: https://github.com/datajoint/element-interface/releases/tag/0.7.0
 [0.6.0]: https://github.com/datajoint/element-interface/releases/tag/0.6.0
 [0.5.4]: https://github.com/datajoint/element-interface/releases/tag/0.5.4
diff --git a/element_interface/version.py b/element_interface/version.py
index 42c688b..70aab85 100644
--- a/element_interface/version.py
+++ b/element_interface/version.py
@@ -1,3 +1,3 @@
 """Package metadata"""
 
-__version__ = "0.8.0"
+__version__ = "0.7.0"