Standardize use of transmit_frequency_start/stop and use pulse_form for BB checks

echopype/calibrate/calibrate_ek.py

@@ -246,9 +246,9 @@ def __init__(
         # Use center frequency if in BB mode, else use nominal channel frequency
         if self.waveform_mode == "BB":
             # use true center frequency to interpolate for various cal params
-            self.freq_center = (beam["frequency_start"] + beam["frequency_end"]).sel(
-                channel=self.chan_sel
-            ) / 2
+            self.freq_center = (
+                beam["transmit_frequency_start"] + beam["transmit_frequency_stop"]
+            ).sel(channel=self.chan_sel) / 2
         else:
             # use nominal channel frequency for CW pulse
             self.freq_center = beam["frequency_nominal"].sel(channel=self.chan_sel)
@@ -309,18 +309,24 @@ def _get_chan_dict(beam: xr.Dataset) -> Dict:
         """
         # Use center frequency for each ping to select BB or CW channels
         # when all samples are encoded as complex samples
-        if "frequency_start" in beam and "frequency_end" in beam:
-            # At least some channels are BB
-            # frequency_start and frequency_end are NaN for CW channels
-            freq_center = (beam["frequency_start"] + beam["frequency_end"]) / 2
-
+        if not np.all(beam["transmit_type"] == "CW"):
+            # At least 1 BB ping exists -- this is analogous to what we had from before
+            # Before: when at least 1 BB ping exists, frequency_start and frequency_end will exist
+
+            # assume transmit_type identical for all pings in a channel
+            first_ping_transmit_type = (
+                beam["transmit_type"].isel(ping_time=0).drop_vars("ping_time")
+            )  # noqa
             return {
-                # For BB: drop channels containing CW samples (nan in freq start/end)
-                "BB": freq_center.dropna(dim="channel").channel,
-                # For CW: drop channels containing BB samples (not nan in freq start/end)
-                "CW": freq_center.where(np.isnan(freq_center), drop=True).channel,
+                # For BB: Keep only non-CW channels (FM or FMD) based on transmit_type
+                "BB": first_ping_transmit_type.where(
+                    first_ping_transmit_type != "CW", drop=True
+                ).channel,  # noqa
+                # For CW: Keep only CW channels based on transmit_type
+                "CW": first_ping_transmit_type.where(
+                    first_ping_transmit_type == "CW", drop=True
+                ).channel,  # noqa
             }
-
         else:
             # All channels are CW
             return {"BB": None, "CW": beam.channel}

echopype/calibrate/ek80_complex.py

@@ -12,22 +12,18 @@ def tapered_chirp(
     fs,
     transmit_duration_nominal,
     slope,
-    frequency_nominal=None,
-    frequency_start=None,
-    frequency_end=None,
+    transmit_frequency_start,
+    transmit_frequency_stop,
 ):
     """
     Create the chirp replica following implementation from Lars Anderson.
 
     Ref source: https://github.com/CRIMAC-WP4-Machine-learning/CRIMAC-Raw-To-Svf-TSf/blob/main/Core/Calculation.py  # noqa
     """
-    if frequency_start is None and frequency_end is None:  # CW waveform
-        frequency_start = frequency_nominal
-        frequency_end = frequency_nominal
 
     tau = transmit_duration_nominal
-    f0 = frequency_start
-    f1 = frequency_end
+    f0 = transmit_frequency_start
+    f1 = transmit_frequency_stop
 
     nsamples = int(np.floor(tau * fs))
     t = np.linspace(0, nsamples - 1, num=nsamples) * 1 / fs
@@ -229,27 +225,20 @@ def get_transmit_signal(
     # Make sure it is BB mode data
     # This is already checked in calibrate_ek
     # but keeping this here for use as standalone function
-    if waveform_mode == "BB" and (("frequency_start" not in beam) or ("frequency_end" not in beam)):
+    if waveform_mode == "BB" and np.all(beam["transmit_type"] == "CW"):
         raise TypeError("File does not contain BB mode complex samples!")
 
     # Generate all transmit replica
     y_all = {}
     y_time_all = {}
+    # TODO: expand to deal with the case with varying tx param across ping_time
+    tx_param_names = [
+        "transmit_duration_nominal",
+        "slope",
+        "transmit_frequency_start",
+        "transmit_frequency_stop",
+    ]
     for ch in beam["channel"].values:
-        # TODO: expand to deal with the case with varying tx param across ping_time
-        if waveform_mode == "BB":
-            tx_param_names = [
-                "transmit_duration_nominal",
-                "slope",
-                "frequency_start",
-                "frequency_end",
-            ]
-        else:
-            tx_param_names = [
-                "transmit_duration_nominal",
-                "slope",
-                "frequency_nominal",
-            ]
         tx_params = {}
         for p in tx_param_names:
             tx_params[p] = np.unique(beam[p].sel(channel=ch))

echopype/convert/set_groups_azfp.py

@@ -382,6 +382,16 @@ def set_beam(self) -> List[xr.Dataset]:
                         "valid_min": 0.0,
                     },
                 ),
+                "transmit_frequency_start": (
+                    ["channel"],
+                    self.freq_sorted,
+                    self._varattrs["beam_var_default"]["transmit_frequency_start"],
+                ),
+                "transmit_frequency_stop": (
+                    ["channel"],
+                    self.freq_sorted,
+                    self._varattrs["beam_var_default"]["transmit_frequency_stop"],
+                ),
                 "beam_stabilisation": (
                     [],
                     np.array(0, np.byte),

echopype/convert/set_groups_ek60.py

@@ -585,6 +585,16 @@ def set_beam(self) -> List[xr.Dataset]:
                     ["channel"],
                     beam_params["gpt_software_version"],
                 ),
+                "transmit_frequency_start": (
+                    ["channel"],
+                    self.freq,
+                    self._varattrs["beam_var_default"]["transmit_frequency_start"],
+                ),
+                "transmit_frequency_stop": (
+                    ["channel"],
+                    self.freq,
+                    self._varattrs["beam_var_default"]["transmit_frequency_stop"],
+                ),
                 "transmit_type": (
                     [],
                     "CW",

echopype/convert/set_groups_ek80.py

@@ -473,7 +473,11 @@ def _assemble_ds_ping_invariant(self, params, data_type):
                         "standard_name": "sound_frequency",
                     },
                 ),
-                "beam_type": (["channel"], beam_params["transducer_beam_type"]),
+                "beam_type": (
+                    ["channel"],
+                    beam_params["transducer_beam_type"],
+                    {"long_name": "type of transducer (0-single, 1-split)"},
+                ),
                 "beamwidth_twoway_alongship": (
                     ["channel"],
                     beam_params["beam_width_alongship"],
@@ -622,15 +626,29 @@ def _assemble_ds_ping_invariant(self, params, data_type):
             attrs={"beam_mode": "vertical", "conversion_equation_t": "type_3"},
         )
 
+        if data_type == "power":
+            ds = ds.assign(
+                {
+                    "transmit_frequency_start": (
+                        ["channel"],
+                        freq,
+                        self._varattrs["beam_var_default"]["transmit_frequency_start"],
+                    ),
+                    "transmit_frequency_stop": (
+                        ["channel"],
+                        freq,
+                        self._varattrs["beam_var_default"]["transmit_frequency_stop"],
+                    ),
+                }
+            )
+
         return ds
 
     def _add_freq_start_end_ds(self, ds_tmp: xr.Dataset, ch: str) -> xr.Dataset:
         """
         Returns a Dataset with variables
-        ``frequency_start`` and ``frequency_end``
-        added to ``ds_tmp`` for a specific channel,
-        if ``frequency_start`` is in ping_data_dict.
-
+        ``transmit_frequency_start`` and ``transmit_frequency_stop``
+        added to ``ds_tmp`` for a specific channel.
         Parameters
         ----------
         ds_tmp: xr.Dataset
@@ -639,54 +657,43 @@ def _add_freq_start_end_ds(self, ds_tmp: xr.Dataset, ch: str) -> xr.Dataset:
             Channel id
         """
 
+        # Process if it's a BB channel (not all pings are CW, where pulse_form encodes CW as 0)
         # CW data encoded as complex samples do NOT have frequency_start and frequency_end
-        # TODO: use PulseForm instead of checking for the existence
-        #   of FrequencyStart and FrequencyEnd
-        if (
-            "frequency_start" in self.parser_obj.ping_data_dict.keys()
-            and self.parser_obj.ping_data_dict["frequency_start"][ch]
-        ):
-            ds_f_start_end = xr.Dataset(
-                {
-                    "frequency_start": (
-                        ["ping_time"],
-                        np.array(
-                            self.parser_obj.ping_data_dict["frequency_start"][ch],
-                            dtype=int,
-                        ),
-                        {
-                            "long_name": "Starting frequency of the transducer",
-                            "units": "Hz",
-                        },
-                    ),
-                    "frequency_end": (
-                        ["ping_time"],
-                        np.array(
-                            self.parser_obj.ping_data_dict["frequency_end"][ch],
-                            dtype=int,
-                        ),
-                        {
-                            "long_name": "Ending frequency of the transducer",
-                            "units": "Hz",
-                        },
-                    ),
-                },
-                coords={
-                    "ping_time": (
-                        ["ping_time"],
-                        self.parser_obj.ping_time[ch],
-                        {
-                            "axis": "T",
-                            "long_name": "Timestamp of each ping",
-                            "standard_name": "time",
-                        },
-                    ),
-                },
-            )
+        if not np.all(np.array(self.parser_obj.ping_data_dict["pulse_form"][ch]) == 0):
+            freq_start = np.array(self.parser_obj.ping_data_dict["frequency_start"][ch])
+            freq_stop = np.array(self.parser_obj.ping_data_dict["frequency_end"][ch])
+        elif not self.sorted_channel["power"]:
+            freq = self.parser_obj.config_datagram["configuration"][ch]["transducer_frequency"]
+            freq_start = np.full(len(self.parser_obj.ping_time[ch]), freq)
+            freq_stop = freq_start
+        else:
+            return ds_tmp
 
-            ds_tmp = xr.merge(
-                [ds_tmp, ds_f_start_end], combine_attrs="override"
-            )  # override keeps the Dataset attributes
+        ds_f_start_end = xr.Dataset(
+            {
+                "transmit_frequency_start": (
+                    ["ping_time"],
+                    freq_start.astype(float),
+                    self._varattrs["beam_var_default"]["transmit_frequency_start"],
+                ),
+                "transmit_frequency_stop": (
+                    ["ping_time"],
+                    freq_stop.astype(float),
+                    self._varattrs["beam_var_default"]["transmit_frequency_stop"],
+                ),
+            },
+            coords={
+                "ping_time": (
+                    ["ping_time"],
+                    self.parser_obj.ping_time[ch],
+                    self._varattrs["beam_coord_default"]["ping_time"],
+                ),
+            },
+        )
+
+        ds_tmp = xr.merge(
+            [ds_tmp, ds_f_start_end], combine_attrs="override"
+        )  # override keeps the Dataset attributes
 
         return ds_tmp
 
@@ -874,6 +881,8 @@ def _assemble_ds_power(self, ch):
                 }
             )
 
+        ds_tmp = self._add_freq_start_end_ds(ds_tmp, ch)
+
         return set_time_encodings(ds_tmp)
 
     def _assemble_ds_common(self, ch, range_sample_size):
@@ -888,6 +897,10 @@ def _assemble_ds_common(self, ch, range_sample_size):
                 self.parser_obj.ping_data_dict["pulse_duration"][ch], dtype="float32"
             )
 
+        def pulse_form_map(pulse_form):
+            str_map = np.array(["CW", "FM", "", "", "", "FMD"])
+            return str_map[pulse_form]
+
         ds_common = xr.Dataset(
             {
                 "sample_interval": (
@@ -928,16 +941,20 @@ def _assemble_ds_common(self, ch, range_sample_size):
                     {
                         "long_name": "Transceiver mode",
                         "flag_values": [0, 1],
-                        "flag_meanings": ["Active", "Inactive"],
+                        "flag_meanings": ["Active", "Unknown"],
                     },
                 ),
-                "pulse_form": (
+                "transmit_type": (
                     ["ping_time"],
-                    np.array(self.parser_obj.ping_data_dict["pulse_form"][ch], dtype=np.byte),
+                    pulse_form_map(np.array(self.parser_obj.ping_data_dict["pulse_form"][ch])),
                     {
-                        "long_name": "Pulse type",
-                        "flag_values": [0, 1, 5],
-                        "flag_meanings": ["CW", "FM", "FMD"],
+                        "long_name": "Type of transmitted pulse",
+                        "flag_values": ["CW", "FM", "FMD"],
+                        "flag_meanings": [
+                            "Continuous Wave",
+                            "Frequency Modulated",
+                            "Frequency Modulated D",
+                        ],
                     },
                 ),
                 "sample_time_offset": (

echopype/echodata/convention/1.0.yml

@@ -75,6 +75,16 @@ variable_and_varattributes:
       long_name: Raw backscatter measurements (real part)
     backscatter_i:
       long_name: Raw backscatter measurements (imaginary part)
+    transmit_frequency_start:
+      long_name: Start frequency in transmitted pulse
+      units: Hz
+      standard_name: sound_frequency
+      valid_min: 0.0
+    transmit_frequency_stop:
+      long_name: Stop frequency in transmitted pulse
+      units: Hz
+      standard_name: sound_frequency
+      valid_min: 0.0
   platform_coord_default:
     time1:
       axis: T