From be04921820e74efdfd9700250089b9810d7baca5 Mon Sep 17 00:00:00 2001 From: Emilio Mayorga Date: Thu, 20 Jul 2023 14:31:28 -0700 Subject: [PATCH 01/20] docs: remove outdated Attention box in Data Processing page --- docs/source/process.rst | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/docs/source/process.rst b/docs/source/process.rst index f2baccbc4..527579e27 100644 --- a/docs/source/process.rst +++ b/docs/source/process.rst @@ -22,17 +22,7 @@ Functionality - EK80 and EA640 broadband echosounders: - Calibration based on pulse compression output in the - form of average over frequency (alpha). - - .. attention:: - This feature is still under development. - We found inconsistencies among pulse compression outputs - from EchoView, Matlab Echolab, and the echopype implementation, see - `#308 `_. - In addition, currently there are issues with calibrating files containing both - broadband and narrowband (the "CW mode") data, see - `#310 `_. - + form of average over frequency. - The same noise removal and MVBS computation functionality available to the narrowband echosounders. From 13f831f07e63acb22029815bfdd4851a1e133d7a Mon Sep 17 00:00:00 2001 From: Emilio Mayorga Date: Tue, 25 Jul 2023 18:47:56 -0700 Subject: [PATCH 02/20] In EK80, rename frequency_start/end to convention-based transmit_frequency_start/stop --- echopype/calibrate/calibrate_ek.py | 12 +++++----- echopype/calibrate/ek80_complex.py | 22 ++++++++++--------- echopype/convert/set_groups_ek80.py | 8 +++---- echopype/echodata/simrad.py | 12 ++++++---- .../calibrate/test_cal_params_integration.py | 2 +- .../tests/calibrate/test_ecs_integration.py | 2 +- 6 files changed, 32 insertions(+), 26 deletions(-) diff --git a/echopype/calibrate/calibrate_ek.py b/echopype/calibrate/calibrate_ek.py index 75e9888f4..17802b239 100644 --- a/echopype/calibrate/calibrate_ek.py +++ b/echopype/calibrate/calibrate_ek.py @@ -246,9 +246,9 @@ def __init__( # Use center frequency if in BB mode, else use nominal channel frequency if self.waveform_mode == "BB": # use true center frequency to interpolate for various cal params - self.freq_center = (beam["frequency_start"] + beam["frequency_end"]).sel( - channel=self.chan_sel - ) / 2 + self.freq_center = ( + beam["transmit_frequency_start"] + beam["transmit_frequency_stop"] + ).sel(channel=self.chan_sel) / 2 else: # use nominal channel frequency for CW pulse self.freq_center = beam["frequency_nominal"].sel(channel=self.chan_sel) @@ -309,10 +309,10 @@ def _get_chan_dict(beam: xr.Dataset) -> Dict: """ # Use center frequency for each ping to select BB or CW channels # when all samples are encoded as complex samples - if "frequency_start" in beam and "frequency_end" in beam: + if "transmit_frequency_start" in beam and "transmit_frequency_stop" in beam: # At least some channels are BB - # frequency_start and frequency_end are NaN for CW channels - freq_center = (beam["frequency_start"] + beam["frequency_end"]) / 2 + # transmit_frequency_start and transmit_frequency_stop are NaN for CW channels + freq_center = (beam["transmit_frequency_start"] + beam["transmit_frequency_stop"]) / 2 return { # For BB: drop channels containing CW samples (nan in freq start/end) diff --git a/echopype/calibrate/ek80_complex.py b/echopype/calibrate/ek80_complex.py index 09670159b..9706c4d6c 100644 --- a/echopype/calibrate/ek80_complex.py +++ b/echopype/calibrate/ek80_complex.py @@ -13,21 +13,21 @@ def tapered_chirp( transmit_duration_nominal, slope, frequency_nominal=None, - frequency_start=None, - frequency_end=None, + transmit_frequency_start=None, + transmit_frequency_stop=None, ): """ Create the chirp replica following implementation from Lars Anderson. Ref source: https://github.com/CRIMAC-WP4-Machine-learning/CRIMAC-Raw-To-Svf-TSf/blob/main/Core/Calculation.py # noqa """ - if frequency_start is None and frequency_end is None: # CW waveform - frequency_start = frequency_nominal - frequency_end = frequency_nominal + if transmit_frequency_start is None and transmit_frequency_stop is None: # CW waveform + transmit_frequency_start = frequency_nominal + transmit_frequency_stop = frequency_nominal tau = transmit_duration_nominal - f0 = frequency_start - f1 = frequency_end + f0 = transmit_frequency_start + f1 = transmit_frequency_stop nsamples = int(np.floor(tau * fs)) t = np.linspace(0, nsamples - 1, num=nsamples) * 1 / fs @@ -229,7 +229,9 @@ def get_transmit_signal( # Make sure it is BB mode data # This is already checked in calibrate_ek # but keeping this here for use as standalone function - if waveform_mode == "BB" and (("frequency_start" not in beam) or ("frequency_end" not in beam)): + if waveform_mode == "BB" and ( + ("transmit_frequency_start" not in beam) or ("transmit_frequency_stop" not in beam) + ): raise TypeError("File does not contain BB mode complex samples!") # Generate all transmit replica @@ -241,8 +243,8 @@ def get_transmit_signal( tx_param_names = [ "transmit_duration_nominal", "slope", - "frequency_start", - "frequency_end", + "transmit_frequency_start", + "transmit_frequency_stop", ] else: tx_param_names = [ diff --git a/echopype/convert/set_groups_ek80.py b/echopype/convert/set_groups_ek80.py index 97d53383e..da1fdd30e 100644 --- a/echopype/convert/set_groups_ek80.py +++ b/echopype/convert/set_groups_ek80.py @@ -608,9 +608,9 @@ def _assemble_ds_ping_invariant(self, params, data_type): def _add_freq_start_end_ds(self, ds_tmp: xr.Dataset, ch: str) -> xr.Dataset: """ Returns a Dataset with variables - ``frequency_start`` and ``frequency_end`` + ``transmit_frequency_start`` and ``transmit_frequency_stop`` added to ``ds_tmp`` for a specific channel, - if ``frequency_start`` is in ping_data_dict. + if ``transmit_frequency_start`` is in ping_data_dict. Parameters ---------- @@ -629,7 +629,7 @@ def _add_freq_start_end_ds(self, ds_tmp: xr.Dataset, ch: str) -> xr.Dataset: ): ds_f_start_end = xr.Dataset( { - "frequency_start": ( + "transmit_frequency_start": ( ["ping_time"], np.array( self.parser_obj.ping_data_dict["frequency_start"][ch], @@ -640,7 +640,7 @@ def _add_freq_start_end_ds(self, ds_tmp: xr.Dataset, ch: str) -> xr.Dataset: "units": "Hz", }, ), - "frequency_end": ( + "transmit_frequency_stop": ( ["ping_time"], np.array( self.parser_obj.ping_data_dict["frequency_end"][ch], diff --git a/echopype/echodata/simrad.py b/echopype/echodata/simrad.py index c63052409..5b9bb4a7c 100644 --- a/echopype/echodata/simrad.py +++ b/echopype/echodata/simrad.py @@ -120,9 +120,12 @@ def _retrieve_correct_beam_group_EK80( if waveform_mode == "BB": # check BB waveform_mode, BB must always have complex data, can have 2 beam groups - # when echodata contains CW power and BB complex samples, and frequency_start + # when echodata contains CW power and BB complex samples, and transmit_frequency_start # variable in Beam_group1 - if waveform_mode == "BB" and "frequency_start" not in echodata["Sonar/Beam_group1"]: + if ( + waveform_mode == "BB" + and "transmit_frequency_start" not in echodata["Sonar/Beam_group1"] + ): raise ValueError("waveform_mode='BB', but broadband data not found!") elif "backscatter_i" not in echodata["Sonar/Beam_group1"].variables: raise ValueError("waveform_mode='BB', but complex data does not exist!") @@ -141,11 +144,12 @@ def _retrieve_correct_beam_group_EK80( # Raise error if waveform_mode="CW" but CW data does not exist if ( encode_mode == "complex" # only check if encode_mode="complex" - and "frequency_start" in echodata["Sonar/Beam_group1"] # only check is data is BB + and "transmit_frequency_start" + in echodata["Sonar/Beam_group1"] # only check is data is BB ): if ( echodata["Sonar/Beam_group1"]["channel"].size # total number of channels - == echodata["Sonar/Beam_group1"]["frequency_start"] + == echodata["Sonar/Beam_group1"]["transmit_frequency_start"] .dropna(dim="channel")["channel"] .size # number of BB channel ): # if all channels are BB diff --git a/echopype/tests/calibrate/test_cal_params_integration.py b/echopype/tests/calibrate/test_cal_params_integration.py index c46d1f02a..e21712641 100644 --- a/echopype/tests/calibrate/test_cal_params_integration.py +++ b/echopype/tests/calibrate/test_cal_params_integration.py @@ -113,7 +113,7 @@ def test_cal_params_intake_EK80_BB_complex(ek80_cal_path): beam = ed["Sonar/Beam_group1"].sel(channel=chan_sel) vend = ed["Vendor_specific"].sel(channel=chan_sel) freq_center = ( - (beam["frequency_start"] + beam["frequency_end"]).sel(channel=chan_sel) / 2) + (beam["transmit_frequency_start"] + beam["transmit_frequency_stop"]).sel(channel=chan_sel) / 2) cal_params_manual = ep.calibrate.cal_params.get_cal_params_EK( "BB", freq_center, beam, vend, {"gain_correction": gain_freq_dep} ) diff --git a/echopype/tests/calibrate/test_ecs_integration.py b/echopype/tests/calibrate/test_ecs_integration.py index 4bf736d2c..82e2d4b4f 100644 --- a/echopype/tests/calibrate/test_ecs_integration.py +++ b/echopype/tests/calibrate/test_ecs_integration.py @@ -153,7 +153,7 @@ def test_ecs_intake_ek80_BB_complex(ek80_path, ecs_path): beam = ed["Sonar/Beam_group1"] chan_w_BB_param = "WBT 549762-15 ES70-7C_ES" freq_center = ( - (beam["frequency_start"] + beam["frequency_end"]) / 2 + (beam["transmit_frequency_start"] + beam["transmit_frequency_stop"]) / 2 ).sel(channel=chan_w_BB_param).drop_vars(["channel"]) for p_name in [ From a7e4e481a29fb3464841ef9de4b64e097e5343f8 Mon Sep 17 00:00:00 2001 From: Emilio Mayorga Date: Tue, 25 Jul 2023 19:08:00 -0700 Subject: [PATCH 03/20] For EK80 transmit_frequency_start/stop, standardize data type (int to float) and attributes --- echopype/convert/set_groups_ek80.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/echopype/convert/set_groups_ek80.py b/echopype/convert/set_groups_ek80.py index da1fdd30e..e976e612d 100644 --- a/echopype/convert/set_groups_ek80.py +++ b/echopype/convert/set_groups_ek80.py @@ -633,22 +633,26 @@ def _add_freq_start_end_ds(self, ds_tmp: xr.Dataset, ch: str) -> xr.Dataset: ["ping_time"], np.array( self.parser_obj.ping_data_dict["frequency_start"][ch], - dtype=int, + dtype=float, ), { - "long_name": "Starting frequency of the transducer", + "long_name": "Start frequency in transmitted pulse", "units": "Hz", + "standard_name": "sound_frequency", + "valid_min": 0.0, }, ), "transmit_frequency_stop": ( ["ping_time"], np.array( self.parser_obj.ping_data_dict["frequency_end"][ch], - dtype=int, + dtype=float, ), { - "long_name": "Ending frequency of the transducer", + "long_name": "Stop frequency in transmitted pulse", "units": "Hz", + "standard_name": "sound_frequency", + "valid_min": 0.0, }, ), }, From 08eeee7d187a65352dd426f40eacbed335b42983 Mon Sep 17 00:00:00 2001 From: Emilio Mayorga Date: Tue, 25 Jul 2023 19:41:59 -0700 Subject: [PATCH 04/20] Add transmit_frequency_start/stop to EK60 & AZFP, set to frequency_nominal --- echopype/convert/set_groups_azfp.py | 20 ++++++++++++++++++++ echopype/convert/set_groups_ek60.py | 20 ++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/echopype/convert/set_groups_azfp.py b/echopype/convert/set_groups_azfp.py index 76da16019..08f50bee5 100644 --- a/echopype/convert/set_groups_azfp.py +++ b/echopype/convert/set_groups_azfp.py @@ -382,6 +382,26 @@ def set_beam(self) -> List[xr.Dataset]: "valid_min": 0.0, }, ), + "transmit_frequency_start": ( + ["channel"], + self.freq_sorted, + { + "long_name": "Start frequency in transmitted pulse", + "units": "Hz", + "standard_name": "sound_frequency", + "valid_min": 0.0, + }, + ), + "transmit_frequency_stop": ( + ["channel"], + self.freq_sorted, + { + "long_name": "Stop frequency in transmitted pulse", + "units": "Hz", + "standard_name": "sound_frequency", + "valid_min": 0.0, + }, + ), }, coords={ "channel": ( diff --git a/echopype/convert/set_groups_ek60.py b/echopype/convert/set_groups_ek60.py index 897a6e55a..01e07b1ce 100644 --- a/echopype/convert/set_groups_ek60.py +++ b/echopype/convert/set_groups_ek60.py @@ -585,6 +585,26 @@ def set_beam(self) -> List[xr.Dataset]: ["channel"], beam_params["gpt_software_version"], ), + "transmit_frequency_start": ( + ["channel"], + self.freq, + { + "long_name": "Start frequency in transmitted pulse", + "units": "Hz", + "standard_name": "sound_frequency", + "valid_min": 0.0, + }, + ), + "transmit_frequency_stop": ( + ["channel"], + self.freq, + { + "long_name": "Stop frequency in transmitted pulse", + "units": "Hz", + "standard_name": "sound_frequency", + "valid_min": 0.0, + }, + ), }, coords={ "channel": ( From 0615af1f232ffb688a2141032c6dac60d21d1b3f Mon Sep 17 00:00:00 2001 From: Emilio Mayorga Date: Thu, 27 Jul 2023 16:09:35 -0700 Subject: [PATCH 05/20] Replace use of frequency_start/stop in ek80 checks for BB channels with pulse_form --- echopype/calibrate/calibrate_ek.py | 4 ++-- echopype/calibrate/ek80_complex.py | 10 ++++++---- echopype/convert/set_groups_ek80.py | 8 ++------ echopype/echodata/simrad.py | 15 ++++++--------- 4 files changed, 16 insertions(+), 21 deletions(-) diff --git a/echopype/calibrate/calibrate_ek.py b/echopype/calibrate/calibrate_ek.py index 17802b239..97a38c897 100644 --- a/echopype/calibrate/calibrate_ek.py +++ b/echopype/calibrate/calibrate_ek.py @@ -309,7 +309,8 @@ def _get_chan_dict(beam: xr.Dataset) -> Dict: """ # Use center frequency for each ping to select BB or CW channels # when all samples are encoded as complex samples - if "transmit_frequency_start" in beam and "transmit_frequency_stop" in beam: + pulse_form_uniq = np.unique(beam["pulse_form"].data) + if 1 in pulse_form_uniq or 5 in pulse_form_uniq: # At least some channels are BB # transmit_frequency_start and transmit_frequency_stop are NaN for CW channels freq_center = (beam["transmit_frequency_start"] + beam["transmit_frequency_stop"]) / 2 @@ -320,7 +321,6 @@ def _get_chan_dict(beam: xr.Dataset) -> Dict: # For CW: drop channels containing BB samples (not nan in freq start/end) "CW": freq_center.where(np.isnan(freq_center), drop=True).channel, } - else: # All channels are CW return {"BB": None, "CW": beam.channel} diff --git a/echopype/calibrate/ek80_complex.py b/echopype/calibrate/ek80_complex.py index 9706c4d6c..10a6b1c01 100644 --- a/echopype/calibrate/ek80_complex.py +++ b/echopype/calibrate/ek80_complex.py @@ -12,6 +12,7 @@ def tapered_chirp( fs, transmit_duration_nominal, slope, + pulse_form, frequency_nominal=None, transmit_frequency_start=None, transmit_frequency_stop=None, @@ -21,7 +22,7 @@ def tapered_chirp( Ref source: https://github.com/CRIMAC-WP4-Machine-learning/CRIMAC-Raw-To-Svf-TSf/blob/main/Core/Calculation.py # noqa """ - if transmit_frequency_start is None and transmit_frequency_stop is None: # CW waveform + if pulse_form[0] == 0: # CW waveform transmit_frequency_start = frequency_nominal transmit_frequency_stop = frequency_nominal @@ -229,9 +230,8 @@ def get_transmit_signal( # Make sure it is BB mode data # This is already checked in calibrate_ek # but keeping this here for use as standalone function - if waveform_mode == "BB" and ( - ("transmit_frequency_start" not in beam) or ("transmit_frequency_stop" not in beam) - ): + pulse_form_uniq = np.unique(beam["pulse_form"].data) + if waveform_mode == "BB" and (len(pulse_form_uniq) == 1 and pulse_form_uniq[0] == 0): raise TypeError("File does not contain BB mode complex samples!") # Generate all transmit replica @@ -243,6 +243,7 @@ def get_transmit_signal( tx_param_names = [ "transmit_duration_nominal", "slope", + "pulse_form", "transmit_frequency_start", "transmit_frequency_stop", ] @@ -250,6 +251,7 @@ def get_transmit_signal( tx_param_names = [ "transmit_duration_nominal", "slope", + "pulse_form", "frequency_nominal", ] tx_params = {} diff --git a/echopype/convert/set_groups_ek80.py b/echopype/convert/set_groups_ek80.py index a4f80d72e..8ac827a89 100644 --- a/echopype/convert/set_groups_ek80.py +++ b/echopype/convert/set_groups_ek80.py @@ -621,12 +621,8 @@ def _add_freq_start_end_ds(self, ds_tmp: xr.Dataset, ch: str) -> xr.Dataset: """ # CW data encoded as complex samples do NOT have frequency_start and frequency_end - # TODO: use PulseForm instead of checking for the existence - # of FrequencyStart and FrequencyEnd - if ( - "frequency_start" in self.parser_obj.ping_data_dict.keys() - and self.parser_obj.ping_data_dict["frequency_start"][ch] - ): + pulse_form_uniq = np.unique(self.parser_obj.ping_data_dict["pulse_form"][ch]) + if 1 in pulse_form_uniq or 5 in pulse_form_uniq: ds_f_start_end = xr.Dataset( { "transmit_frequency_start": ( diff --git a/echopype/echodata/simrad.py b/echopype/echodata/simrad.py index 5b9bb4a7c..0e47613e8 100644 --- a/echopype/echodata/simrad.py +++ b/echopype/echodata/simrad.py @@ -3,6 +3,8 @@ """ from typing import Optional, Tuple +import numpy as np + from .echodata import EchoData @@ -118,14 +120,12 @@ def _retrieve_correct_beam_group_EK80( power_ed_group = None complex_ed_group = None + pulse_form_uniq = np.unique(echodata["Sonar/Beam_group1"]["pulse_form"].data) if waveform_mode == "BB": # check BB waveform_mode, BB must always have complex data, can have 2 beam groups # when echodata contains CW power and BB complex samples, and transmit_frequency_start # variable in Beam_group1 - if ( - waveform_mode == "BB" - and "transmit_frequency_start" not in echodata["Sonar/Beam_group1"] - ): + if waveform_mode == "BB" and (len(pulse_form_uniq) == 1 and pulse_form_uniq[0] == 0): raise ValueError("waveform_mode='BB', but broadband data not found!") elif "backscatter_i" not in echodata["Sonar/Beam_group1"].variables: raise ValueError("waveform_mode='BB', but complex data does not exist!") @@ -142,11 +142,8 @@ def _retrieve_correct_beam_group_EK80( # 3) power samples are in Sonar/Beam_group2 if two beam groups exist # Raise error if waveform_mode="CW" but CW data does not exist - if ( - encode_mode == "complex" # only check if encode_mode="complex" - and "transmit_frequency_start" - in echodata["Sonar/Beam_group1"] # only check is data is BB - ): + if encode_mode == "complex" and (1 in pulse_form_uniq or 5 in pulse_form_uniq): + # complex + BB data if ( echodata["Sonar/Beam_group1"]["channel"].size # total number of channels == echodata["Sonar/Beam_group1"]["transmit_frequency_start"] From 5e9287f21cfeaae63f2cd41d8c1d540495d5da30 Mon Sep 17 00:00:00 2001 From: Emilio Mayorga Date: Fri, 28 Jul 2023 01:09:59 -0700 Subject: [PATCH 06/20] Rename pulse_form to transmit_type per convention, and map int codes to strings (CW, FM, FMD) --- echopype/convert/set_groups_ek80.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/echopype/convert/set_groups_ek80.py b/echopype/convert/set_groups_ek80.py index a4f80d72e..3525d6b3f 100644 --- a/echopype/convert/set_groups_ek80.py +++ b/echopype/convert/set_groups_ek80.py @@ -873,6 +873,10 @@ def _assemble_ds_common(self, ch, range_sample_size): self.parser_obj.ping_data_dict["pulse_duration"][ch], dtype="float32" ) + def pulse_form_map(pulse_form): + str_map = np.array(["CW", "FM", "", "", "", "FMD"]) + return str_map[pulse_form] + ds_common = xr.Dataset( { "sample_interval": ( @@ -916,19 +920,23 @@ def _assemble_ds_common(self, ch, range_sample_size): "flag_meanings": ["Active", "Inactive"], }, ), - "pulse_form": ( + "transmit_type": ( ["ping_time"], - np.array(self.parser_obj.ping_data_dict["pulse_form"][ch], dtype=np.byte), - { - "long_name": "Pulse type", - "flag_values": [0, 1, 5], - "flag_meanings": ["CW", "FM", "FMD"], - }, + pulse_form_map(np.array(self.parser_obj.ping_data_dict["pulse_form"][ch])), + {"long_name": "Type of transmitted pulse"}, ), "range_sample_offset": ( ["ping_time"], np.array(self.parser_obj.ping_data_dict["offset"][ch], dtype=int), - {"long_name": "First sample number"}, + { + "long_name": "First sample number", + "flag_values": ["CW", "FM", "FMD"], + "flag_meanings": [ + "Continuous Wave", + "Frequency Modulated", + "Frequency Modulated D", + ], + }, ), }, coords={ From 22fe80faad9887bd5836f91f1be3620f73caba2a Mon Sep 17 00:00:00 2001 From: Emilio Mayorga Date: Fri, 28 Jul 2023 01:32:15 -0700 Subject: [PATCH 07/20] Assigned attributes to the wrong variable, in previous commit --- echopype/convert/set_groups_ek80.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/echopype/convert/set_groups_ek80.py b/echopype/convert/set_groups_ek80.py index 3525d6b3f..1cceeacc2 100644 --- a/echopype/convert/set_groups_ek80.py +++ b/echopype/convert/set_groups_ek80.py @@ -923,13 +923,8 @@ def pulse_form_map(pulse_form): "transmit_type": ( ["ping_time"], pulse_form_map(np.array(self.parser_obj.ping_data_dict["pulse_form"][ch])), - {"long_name": "Type of transmitted pulse"}, - ), - "range_sample_offset": ( - ["ping_time"], - np.array(self.parser_obj.ping_data_dict["offset"][ch], dtype=int), { - "long_name": "First sample number", + "long_name": "Type of transmitted pulse", "flag_values": ["CW", "FM", "FMD"], "flag_meanings": [ "Continuous Wave", @@ -938,6 +933,11 @@ def pulse_form_map(pulse_form): ], }, ), + "range_sample_offset": ( + ["ping_time"], + np.array(self.parser_obj.ping_data_dict["offset"][ch], dtype=int), + {"long_name": "First sample number"}, + ), }, coords={ "ping_time": ( From 89d1c02c04227442256814b2896ac68b74445711 Mon Sep 17 00:00:00 2001 From: Emilio Mayorga Date: Fri, 28 Jul 2023 13:33:50 -0700 Subject: [PATCH 08/20] The meaning for ek80 channel_mode = 1 is not described in ek80 manual --- echopype/convert/set_groups_ek80.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/echopype/convert/set_groups_ek80.py b/echopype/convert/set_groups_ek80.py index 1cceeacc2..53de2a91c 100644 --- a/echopype/convert/set_groups_ek80.py +++ b/echopype/convert/set_groups_ek80.py @@ -917,7 +917,7 @@ def pulse_form_map(pulse_form): { "long_name": "Transceiver mode", "flag_values": [0, 1], - "flag_meanings": ["Active", "Inactive"], + "flag_meanings": ["Active", "Unknown"], }, ), "transmit_type": ( From 589a50c15bd9f6374e9557109a7e548676589b07 Mon Sep 17 00:00:00 2001 From: Emilio Mayorga Date: Sat, 29 Jul 2023 13:42:02 -0700 Subject: [PATCH 09/20] Ensure variables remain encoded as int types even after xr.merge --- echopype/utils/coding.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/echopype/utils/coding.py b/echopype/utils/coding.py index 61d11714a..a4a57a7cc 100644 --- a/echopype/utils/coding.py +++ b/echopype/utils/coding.py @@ -42,6 +42,8 @@ "channel": np.str_, "cal_channel_id": np.str_, "beam": np.str_, + "channel_mode": np.byte, + "range_sample_offset": np.int32, } # channel name # beam name PREFERRED_CHUNKS = "preferred_chunks" From b47c170ba5e415c5a3a37c44f086fa457ee9869b Mon Sep 17 00:00:00 2001 From: Emilio Mayorga Date: Sat, 29 Jul 2023 13:44:54 -0700 Subject: [PATCH 10/20] Overhaul EK80 checks for BB vs CW to use pulse_form / transmit_type --- echopype/calibrate/calibrate_ek.py | 6 +++--- echopype/calibrate/ek80_complex.py | 11 +++++------ echopype/convert/set_groups_ek80.py | 6 +++--- echopype/echodata/simrad.py | 6 +++--- 4 files changed, 14 insertions(+), 15 deletions(-) diff --git a/echopype/calibrate/calibrate_ek.py b/echopype/calibrate/calibrate_ek.py index 97a38c897..53f271d39 100644 --- a/echopype/calibrate/calibrate_ek.py +++ b/echopype/calibrate/calibrate_ek.py @@ -309,9 +309,9 @@ def _get_chan_dict(beam: xr.Dataset) -> Dict: """ # Use center frequency for each ping to select BB or CW channels # when all samples are encoded as complex samples - pulse_form_uniq = np.unique(beam["pulse_form"].data) - if 1 in pulse_form_uniq or 5 in pulse_form_uniq: - # At least some channels are BB + if not np.all(beam["transmit_type"] == "CW"): + # At least 1 BB ping exists -- this is analogous to what we had from before + # Before: when at least 1 BB ping exists, frequency_start and frequency_end will exist # transmit_frequency_start and transmit_frequency_stop are NaN for CW channels freq_center = (beam["transmit_frequency_start"] + beam["transmit_frequency_stop"]) / 2 diff --git a/echopype/calibrate/ek80_complex.py b/echopype/calibrate/ek80_complex.py index 10a6b1c01..4fe90cc03 100644 --- a/echopype/calibrate/ek80_complex.py +++ b/echopype/calibrate/ek80_complex.py @@ -12,7 +12,7 @@ def tapered_chirp( fs, transmit_duration_nominal, slope, - pulse_form, + transmit_type, frequency_nominal=None, transmit_frequency_start=None, transmit_frequency_stop=None, @@ -22,7 +22,7 @@ def tapered_chirp( Ref source: https://github.com/CRIMAC-WP4-Machine-learning/CRIMAC-Raw-To-Svf-TSf/blob/main/Core/Calculation.py # noqa """ - if pulse_form[0] == 0: # CW waveform + if transmit_type == "CW": # CW waveform transmit_frequency_start = frequency_nominal transmit_frequency_stop = frequency_nominal @@ -230,8 +230,7 @@ def get_transmit_signal( # Make sure it is BB mode data # This is already checked in calibrate_ek # but keeping this here for use as standalone function - pulse_form_uniq = np.unique(beam["pulse_form"].data) - if waveform_mode == "BB" and (len(pulse_form_uniq) == 1 and pulse_form_uniq[0] == 0): + if waveform_mode == "BB" and np.all(beam["transmit_type"] == "CW"): raise TypeError("File does not contain BB mode complex samples!") # Generate all transmit replica @@ -243,7 +242,7 @@ def get_transmit_signal( tx_param_names = [ "transmit_duration_nominal", "slope", - "pulse_form", + "transmit_type", "transmit_frequency_start", "transmit_frequency_stop", ] @@ -251,7 +250,7 @@ def get_transmit_signal( tx_param_names = [ "transmit_duration_nominal", "slope", - "pulse_form", + "transmit_type", "frequency_nominal", ] tx_params = {} diff --git a/echopype/convert/set_groups_ek80.py b/echopype/convert/set_groups_ek80.py index 1b9d22024..fef1011fe 100644 --- a/echopype/convert/set_groups_ek80.py +++ b/echopype/convert/set_groups_ek80.py @@ -620,9 +620,9 @@ def _add_freq_start_end_ds(self, ds_tmp: xr.Dataset, ch: str) -> xr.Dataset: Channel id """ + # Process if it's a BB channel (not all pings are CW, where pulse_form encodes CW as 0) # CW data encoded as complex samples do NOT have frequency_start and frequency_end - pulse_form_uniq = np.unique(self.parser_obj.ping_data_dict["pulse_form"][ch]) - if 1 in pulse_form_uniq or 5 in pulse_form_uniq: + if not np.all(np.array(self.parser_obj.ping_data_dict["pulse_form"][ch]) == 0): ds_f_start_end = xr.Dataset( { "transmit_frequency_start": ( @@ -931,7 +931,7 @@ def pulse_form_map(pulse_form): ), "range_sample_offset": ( ["ping_time"], - np.array(self.parser_obj.ping_data_dict["offset"][ch], dtype=int), + np.array(self.parser_obj.ping_data_dict["offset"][ch], dtype=np.int32), {"long_name": "First sample number"}, ), }, diff --git a/echopype/echodata/simrad.py b/echopype/echodata/simrad.py index 0e47613e8..914a2e911 100644 --- a/echopype/echodata/simrad.py +++ b/echopype/echodata/simrad.py @@ -120,12 +120,12 @@ def _retrieve_correct_beam_group_EK80( power_ed_group = None complex_ed_group = None - pulse_form_uniq = np.unique(echodata["Sonar/Beam_group1"]["pulse_form"].data) + transmit_type_has_BB = not np.all(echodata["Sonar/Beam_group1"]["transmit_type"] == "CW") if waveform_mode == "BB": # check BB waveform_mode, BB must always have complex data, can have 2 beam groups # when echodata contains CW power and BB complex samples, and transmit_frequency_start # variable in Beam_group1 - if waveform_mode == "BB" and (len(pulse_form_uniq) == 1 and pulse_form_uniq[0] == 0): + if not transmit_type_has_BB: raise ValueError("waveform_mode='BB', but broadband data not found!") elif "backscatter_i" not in echodata["Sonar/Beam_group1"].variables: raise ValueError("waveform_mode='BB', but complex data does not exist!") @@ -142,7 +142,7 @@ def _retrieve_correct_beam_group_EK80( # 3) power samples are in Sonar/Beam_group2 if two beam groups exist # Raise error if waveform_mode="CW" but CW data does not exist - if encode_mode == "complex" and (1 in pulse_form_uniq or 5 in pulse_form_uniq): + if encode_mode == "complex" and transmit_type_has_BB: # complex + BB data if ( echodata["Sonar/Beam_group1"]["channel"].size # total number of channels From 6a5eccef1e83c7c30b5025ad7f2e55162a6a93a3 Mon Sep 17 00:00:00 2001 From: Emilio Mayorga Date: Sat, 29 Jul 2023 15:49:13 -0700 Subject: [PATCH 11/20] Set transmit_frequency_start/stop attributes in 1.0.yml and update set_groups_azfp/ek60 to use that --- echopype/convert/set_groups_azfp.py | 14 ++------------ echopype/convert/set_groups_ek60.py | 14 ++------------ echopype/echodata/convention/1.0.yml | 10 ++++++++++ 3 files changed, 14 insertions(+), 24 deletions(-) diff --git a/echopype/convert/set_groups_azfp.py b/echopype/convert/set_groups_azfp.py index 08f50bee5..e47381c79 100644 --- a/echopype/convert/set_groups_azfp.py +++ b/echopype/convert/set_groups_azfp.py @@ -385,22 +385,12 @@ def set_beam(self) -> List[xr.Dataset]: "transmit_frequency_start": ( ["channel"], self.freq_sorted, - { - "long_name": "Start frequency in transmitted pulse", - "units": "Hz", - "standard_name": "sound_frequency", - "valid_min": 0.0, - }, + self._varattrs["beam_var_default"]["transmit_frequency_start"], ), "transmit_frequency_stop": ( ["channel"], self.freq_sorted, - { - "long_name": "Stop frequency in transmitted pulse", - "units": "Hz", - "standard_name": "sound_frequency", - "valid_min": 0.0, - }, + self._varattrs["beam_var_default"]["transmit_frequency_stop"], ), }, coords={ diff --git a/echopype/convert/set_groups_ek60.py b/echopype/convert/set_groups_ek60.py index b67a81d78..d55ed760c 100644 --- a/echopype/convert/set_groups_ek60.py +++ b/echopype/convert/set_groups_ek60.py @@ -588,22 +588,12 @@ def set_beam(self) -> List[xr.Dataset]: "transmit_frequency_start": ( ["channel"], self.freq, - { - "long_name": "Start frequency in transmitted pulse", - "units": "Hz", - "standard_name": "sound_frequency", - "valid_min": 0.0, - }, + self._varattrs["beam_var_default"]["transmit_frequency_start"], ), "transmit_frequency_stop": ( ["channel"], self.freq, - { - "long_name": "Stop frequency in transmitted pulse", - "units": "Hz", - "standard_name": "sound_frequency", - "valid_min": 0.0, - }, + self._varattrs["beam_var_default"]["transmit_frequency_stop"], ), }, coords={ diff --git a/echopype/echodata/convention/1.0.yml b/echopype/echodata/convention/1.0.yml index d0453f998..66068300b 100644 --- a/echopype/echodata/convention/1.0.yml +++ b/echopype/echodata/convention/1.0.yml @@ -75,6 +75,16 @@ variable_and_varattributes: long_name: Raw backscatter measurements (real part) backscatter_i: long_name: Raw backscatter measurements (imaginary part) + transmit_frequency_start: + long_name: Start frequency in transmitted pulse + units: Hz + standard_name: sound_frequency + valid_min: 0.0 + transmit_frequency_stop: + long_name: Stop frequency in transmitted pulse + units: Hz + standard_name: sound_frequency + valid_min: 0.0 platform_coord_default: time1: axis: T From 5a8e0d956832a26fdd3a139130ce7f852a8a1593 Mon Sep 17 00:00:00 2001 From: Emilio Mayorga Date: Sat, 29 Jul 2023 20:34:37 -0700 Subject: [PATCH 12/20] For ek80 transmit_freq_start/stop is now created and/or populated with frequency_nominal for power-only channels. Updated all BB vs CW checks. --- echopype/calibrate/calibrate_ek.py | 8 +-- echopype/convert/set_groups_ek80.py | 106 +++++++++++++++------------- echopype/echodata/simrad.py | 9 +-- 3 files changed, 64 insertions(+), 59 deletions(-) diff --git a/echopype/calibrate/calibrate_ek.py b/echopype/calibrate/calibrate_ek.py index 53f271d39..78202e81b 100644 --- a/echopype/calibrate/calibrate_ek.py +++ b/echopype/calibrate/calibrate_ek.py @@ -316,10 +316,10 @@ def _get_chan_dict(beam: xr.Dataset) -> Dict: freq_center = (beam["transmit_frequency_start"] + beam["transmit_frequency_stop"]) / 2 return { - # For BB: drop channels containing CW samples (nan in freq start/end) - "BB": freq_center.dropna(dim="channel").channel, - # For CW: drop channels containing BB samples (not nan in freq start/end) - "CW": freq_center.where(np.isnan(freq_center), drop=True).channel, + # For BB: Keep only non-CW channels (FM or FMD) based on transmit_type + "BB": freq_center.where(beam["transmit_type"] != "CW", drop=True).channel, + # For CW: Keep only CW channels based on transmit_type + "CW": freq_center.where(beam["transmit_type"] == "CW", drop=True).channel, } else: # All channels are CW diff --git a/echopype/convert/set_groups_ek80.py b/echopype/convert/set_groups_ek80.py index fef1011fe..9afaa0cdf 100644 --- a/echopype/convert/set_groups_ek80.py +++ b/echopype/convert/set_groups_ek80.py @@ -473,7 +473,11 @@ def _assemble_ds_ping_invariant(self, params, data_type): "standard_name": "sound_frequency", }, ), - "beam_type": (["channel"], beam_params["transducer_beam_type"]), + "beam_type": ( + ["channel"], + beam_params["transducer_beam_type"], + {"long_name": "type of transducer (0-single, 1-split)"}, + ), "beamwidth_twoway_alongship": ( ["channel"], beam_params["beam_width_alongship"], @@ -603,15 +607,29 @@ def _assemble_ds_ping_invariant(self, params, data_type): attrs={"beam_mode": "vertical", "conversion_equation_t": "type_3"}, ) + if data_type == "power": + ds = ds.assign( + { + "transmit_frequency_start": ( + ["channel"], + freq, + self._varattrs["beam_var_default"]["transmit_frequency_start"], + ), + "transmit_frequency_stop": ( + ["channel"], + freq, + self._varattrs["beam_var_default"]["transmit_frequency_stop"], + ), + } + ) + return ds def _add_freq_start_end_ds(self, ds_tmp: xr.Dataset, ch: str) -> xr.Dataset: """ Returns a Dataset with variables ``transmit_frequency_start`` and ``transmit_frequency_stop`` - added to ``ds_tmp`` for a specific channel, - if ``transmit_frequency_start`` is in ping_data_dict. - + added to ``ds_tmp`` for a specific channel. Parameters ---------- ds_tmp: xr.Dataset @@ -623,51 +641,41 @@ def _add_freq_start_end_ds(self, ds_tmp: xr.Dataset, ch: str) -> xr.Dataset: # Process if it's a BB channel (not all pings are CW, where pulse_form encodes CW as 0) # CW data encoded as complex samples do NOT have frequency_start and frequency_end if not np.all(np.array(self.parser_obj.ping_data_dict["pulse_form"][ch]) == 0): - ds_f_start_end = xr.Dataset( - { - "transmit_frequency_start": ( - ["ping_time"], - np.array( - self.parser_obj.ping_data_dict["frequency_start"][ch], - dtype=float, - ), - { - "long_name": "Start frequency in transmitted pulse", - "units": "Hz", - "standard_name": "sound_frequency", - "valid_min": 0.0, - }, - ), - "transmit_frequency_stop": ( - ["ping_time"], - np.array( - self.parser_obj.ping_data_dict["frequency_end"][ch], - dtype=float, - ), - { - "long_name": "Stop frequency in transmitted pulse", - "units": "Hz", - "standard_name": "sound_frequency", - "valid_min": 0.0, - }, - ), - }, - coords={ - "ping_time": ( - ["ping_time"], - self.parser_obj.ping_time[ch], - { - "axis": "T", - "long_name": "Timestamp of each ping", - "standard_name": "time", - }, - ), - }, - ) + freq_start = np.array(self.parser_obj.ping_data_dict["frequency_start"][ch]) + freq_stop = np.array(self.parser_obj.ping_data_dict["frequency_end"][ch]) + elif not self.sorted_channel["power"]: + freq = self.parser_obj.config_datagram["configuration"][ch]["transducer_frequency"] + # freq_start = np.full(self.parser_obj.ping_data_dict["power"][ch], freq) + freq_start = np.full(len(self.parser_obj.ping_time[ch]), freq) + freq_stop = freq_start + else: + return ds_tmp - ds_tmp = xr.merge( - [ds_tmp, ds_f_start_end], combine_attrs="override" - ) # override keeps the Dataset attributes + ds_f_start_end = xr.Dataset( + { + "transmit_frequency_start": ( + ["ping_time"], + freq_start.astype(float), + self._varattrs["beam_var_default"]["transmit_frequency_start"], + ), + "transmit_frequency_stop": ( + ["ping_time"], + freq_stop.astype(float), + self._varattrs["beam_var_default"]["transmit_frequency_stop"], + ), + }, + coords={ + "ping_time": ( + ["ping_time"], + self.parser_obj.ping_time[ch], + self._varattrs["beam_coord_default"]["ping_time"], + ), + }, + ) + + ds_tmp = xr.merge( + [ds_tmp, ds_f_start_end], combine_attrs="override" + ) # override keeps the Dataset attributes return ds_tmp @@ -855,6 +863,8 @@ def _assemble_ds_power(self, ch): } ) + ds_tmp = self._add_freq_start_end_ds(ds_tmp, ch) + return set_time_encodings(ds_tmp) def _assemble_ds_common(self, ch, range_sample_size): diff --git a/echopype/echodata/simrad.py b/echopype/echodata/simrad.py index 914a2e911..a3d4e1055 100644 --- a/echopype/echodata/simrad.py +++ b/echopype/echodata/simrad.py @@ -143,13 +143,8 @@ def _retrieve_correct_beam_group_EK80( # Raise error if waveform_mode="CW" but CW data does not exist if encode_mode == "complex" and transmit_type_has_BB: - # complex + BB data - if ( - echodata["Sonar/Beam_group1"]["channel"].size # total number of channels - == echodata["Sonar/Beam_group1"]["transmit_frequency_start"] - .dropna(dim="channel")["channel"] - .size # number of BB channel - ): # if all channels are BB + # complex + BB data (all channels are BB) + if np.all(echodata["Sonar/Beam_group1"]["transmit_type"] != "CW"): raise ValueError("waveform_mode='CW', but all data are broadband (BB)!") if echodata["Sonar/Beam_group2"] is None: From cc84cce5d96e8710f5af55dfb6bb074e13108e22 Mon Sep 17 00:00:00 2001 From: Emilio Mayorga Date: Sun, 30 Jul 2023 12:11:48 -0700 Subject: [PATCH 13/20] Simplified ek80 tapered_chirp now that CW channels always contain freq nominal in transmit_frequency_start/stop --- echopype/calibrate/ek80_complex.py | 32 ++++++++++-------------------- 1 file changed, 10 insertions(+), 22 deletions(-) diff --git a/echopype/calibrate/ek80_complex.py b/echopype/calibrate/ek80_complex.py index 4fe90cc03..63a10fa41 100644 --- a/echopype/calibrate/ek80_complex.py +++ b/echopype/calibrate/ek80_complex.py @@ -13,18 +13,14 @@ def tapered_chirp( transmit_duration_nominal, slope, transmit_type, - frequency_nominal=None, - transmit_frequency_start=None, - transmit_frequency_stop=None, + transmit_frequency_start, + transmit_frequency_stop, ): """ Create the chirp replica following implementation from Lars Anderson. Ref source: https://github.com/CRIMAC-WP4-Machine-learning/CRIMAC-Raw-To-Svf-TSf/blob/main/Core/Calculation.py # noqa """ - if transmit_type == "CW": # CW waveform - transmit_frequency_start = frequency_nominal - transmit_frequency_stop = frequency_nominal tau = transmit_duration_nominal f0 = transmit_frequency_start @@ -236,23 +232,15 @@ def get_transmit_signal( # Generate all transmit replica y_all = {} y_time_all = {} + # TODO: expand to deal with the case with varying tx param across ping_time + tx_param_names = [ + "transmit_duration_nominal", + "slope", + "transmit_type", + "transmit_frequency_start", + "transmit_frequency_stop", + ] for ch in beam["channel"].values: - # TODO: expand to deal with the case with varying tx param across ping_time - if waveform_mode == "BB": - tx_param_names = [ - "transmit_duration_nominal", - "slope", - "transmit_type", - "transmit_frequency_start", - "transmit_frequency_stop", - ] - else: - tx_param_names = [ - "transmit_duration_nominal", - "slope", - "transmit_type", - "frequency_nominal", - ] tx_params = {} for p in tx_param_names: tx_params[p] = np.unique(beam[p].sel(channel=ch)) From 9bec93a9707d95fb717f613864d90bb8c9531920 Mon Sep 17 00:00:00 2001 From: Emilio Mayorga Date: Wed, 2 Aug 2023 21:49:07 -0700 Subject: [PATCH 14/20] Small cleanups to comments, unused function parameter, etc, all involving ek80 --- echopype/calibrate/ek80_complex.py | 3 +-- echopype/convert/set_groups_ek80.py | 1 - echopype/echodata/simrad.py | 3 +-- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/echopype/calibrate/ek80_complex.py b/echopype/calibrate/ek80_complex.py index 63a10fa41..7d53718cf 100644 --- a/echopype/calibrate/ek80_complex.py +++ b/echopype/calibrate/ek80_complex.py @@ -12,7 +12,6 @@ def tapered_chirp( fs, transmit_duration_nominal, slope, - transmit_type, transmit_frequency_start, transmit_frequency_stop, ): @@ -236,7 +235,7 @@ def get_transmit_signal( tx_param_names = [ "transmit_duration_nominal", "slope", - "transmit_type", + "transmit_type", # not needed for tapered_chirp but good to check uniqueness "transmit_frequency_start", "transmit_frequency_stop", ] diff --git a/echopype/convert/set_groups_ek80.py b/echopype/convert/set_groups_ek80.py index 9afaa0cdf..078cd4c50 100644 --- a/echopype/convert/set_groups_ek80.py +++ b/echopype/convert/set_groups_ek80.py @@ -645,7 +645,6 @@ def _add_freq_start_end_ds(self, ds_tmp: xr.Dataset, ch: str) -> xr.Dataset: freq_stop = np.array(self.parser_obj.ping_data_dict["frequency_end"][ch]) elif not self.sorted_channel["power"]: freq = self.parser_obj.config_datagram["configuration"][ch]["transducer_frequency"] - # freq_start = np.full(self.parser_obj.ping_data_dict["power"][ch], freq) freq_start = np.full(len(self.parser_obj.ping_time[ch]), freq) freq_stop = freq_start else: diff --git a/echopype/echodata/simrad.py b/echopype/echodata/simrad.py index a3d4e1055..61720539c 100644 --- a/echopype/echodata/simrad.py +++ b/echopype/echodata/simrad.py @@ -123,8 +123,7 @@ def _retrieve_correct_beam_group_EK80( transmit_type_has_BB = not np.all(echodata["Sonar/Beam_group1"]["transmit_type"] == "CW") if waveform_mode == "BB": # check BB waveform_mode, BB must always have complex data, can have 2 beam groups - # when echodata contains CW power and BB complex samples, and transmit_frequency_start - # variable in Beam_group1 + # when echodata contains CW power and BB complex samples if not transmit_type_has_BB: raise ValueError("waveform_mode='BB', but broadband data not found!") elif "backscatter_i" not in echodata["Sonar/Beam_group1"].variables: From 2a1950ddcc1715630f61b8b3b262d0cadf5d8ad6 Mon Sep 17 00:00:00 2001 From: Emilio Mayorga Date: Wed, 2 Aug 2023 23:25:34 -0700 Subject: [PATCH 15/20] Simplify ek80 test for encode_mode = 'complex' but there are no CW pings --- echopype/echodata/simrad.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/echopype/echodata/simrad.py b/echopype/echodata/simrad.py index 61720539c..761e0fa35 100644 --- a/echopype/echodata/simrad.py +++ b/echopype/echodata/simrad.py @@ -120,11 +120,10 @@ def _retrieve_correct_beam_group_EK80( power_ed_group = None complex_ed_group = None - transmit_type_has_BB = not np.all(echodata["Sonar/Beam_group1"]["transmit_type"] == "CW") if waveform_mode == "BB": # check BB waveform_mode, BB must always have complex data, can have 2 beam groups # when echodata contains CW power and BB complex samples - if not transmit_type_has_BB: + if np.all(echodata["Sonar/Beam_group1"]["transmit_type"] == "CW"): raise ValueError("waveform_mode='BB', but broadband data not found!") elif "backscatter_i" not in echodata["Sonar/Beam_group1"].variables: raise ValueError("waveform_mode='BB', but complex data does not exist!") @@ -133,18 +132,17 @@ def _retrieve_correct_beam_group_EK80( complex_ed_group = "Sonar/Beam_group1" else: complex_ed_group = "Sonar/Beam_group1" - else: # CW can have complex or power data, so we just need to make sure that # 1) complex samples always exist in Sonar/Beam_group1 # 2) power samples are in Sonar/Beam_group1 if only one beam group exists # 3) power samples are in Sonar/Beam_group2 if two beam groups exist - # Raise error if waveform_mode="CW" but CW data does not exist - if encode_mode == "complex" and transmit_type_has_BB: - # complex + BB data (all channels are BB) - if np.all(echodata["Sonar/Beam_group1"]["transmit_type"] != "CW"): - raise ValueError("waveform_mode='CW', but all data are broadband (BB)!") + # Raise error if waveform_mode="CW" but CW data does not exist (not a single ping is CW) + if encode_mode == "complex" and np.all( + echodata["Sonar/Beam_group1"]["transmit_type"] != "CW" + ): + raise ValueError("waveform_mode='CW', but all data are broadband (BB)!") if echodata["Sonar/Beam_group2"] is None: if encode_mode == "power": From b6afd767312d28c23bebe4b05c0ca59b5545bb55 Mon Sep 17 00:00:00 2001 From: Emilio Mayorga Date: Thu, 3 Aug 2023 00:30:06 -0700 Subject: [PATCH 16/20] Remove transmit_type from tx_param_names b/c it's no longer an argument in tapered_chirp --- echopype/calibrate/ek80_complex.py | 1 - 1 file changed, 1 deletion(-) diff --git a/echopype/calibrate/ek80_complex.py b/echopype/calibrate/ek80_complex.py index 7d53718cf..6e6fefe1f 100644 --- a/echopype/calibrate/ek80_complex.py +++ b/echopype/calibrate/ek80_complex.py @@ -235,7 +235,6 @@ def get_transmit_signal( tx_param_names = [ "transmit_duration_nominal", "slope", - "transmit_type", # not needed for tapered_chirp but good to check uniqueness "transmit_frequency_start", "transmit_frequency_stop", ] From c199775e5d74d80c7d8325f2f67af8bb724de03a Mon Sep 17 00:00:00 2001 From: Emilio Mayorga Date: Thu, 3 Aug 2023 13:50:19 -0700 Subject: [PATCH 17/20] Optimize ek80 BB vs CW tests using transmit_type on first ping_time --- echopype/calibrate/calibrate_ek.py | 14 ++++++++++---- echopype/echodata/simrad.py | 11 +++++------ 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/echopype/calibrate/calibrate_ek.py b/echopype/calibrate/calibrate_ek.py index 78202e81b..591aa4fdb 100644 --- a/echopype/calibrate/calibrate_ek.py +++ b/echopype/calibrate/calibrate_ek.py @@ -312,14 +312,20 @@ def _get_chan_dict(beam: xr.Dataset) -> Dict: if not np.all(beam["transmit_type"] == "CW"): # At least 1 BB ping exists -- this is analogous to what we had from before # Before: when at least 1 BB ping exists, frequency_start and frequency_end will exist - # transmit_frequency_start and transmit_frequency_stop are NaN for CW channels - freq_center = (beam["transmit_frequency_start"] + beam["transmit_frequency_stop"]) / 2 + # assume transmit_type identical for all pings in a channel + first_ping_transmit_type = ( + beam["transmit_type"].isel(ping_time=0).drop_vars("ping_time") + ) # noqa return { # For BB: Keep only non-CW channels (FM or FMD) based on transmit_type - "BB": freq_center.where(beam["transmit_type"] != "CW", drop=True).channel, + "BB": first_ping_transmit_type.where( + first_ping_transmit_type != "CW", drop=True + ).channel, # noqa # For CW: Keep only CW channels based on transmit_type - "CW": freq_center.where(beam["transmit_type"] == "CW", drop=True).channel, + "CW": first_ping_transmit_type.where( + first_ping_transmit_type == "CW", drop=True + ).channel, # noqa } else: # All channels are CW diff --git a/echopype/echodata/simrad.py b/echopype/echodata/simrad.py index 761e0fa35..fb6693b24 100644 --- a/echopype/echodata/simrad.py +++ b/echopype/echodata/simrad.py @@ -120,12 +120,13 @@ def _retrieve_correct_beam_group_EK80( power_ed_group = None complex_ed_group = None + transmit_type = echodata["Sonar/Beam_group1"]["transmit_type"] + # assume transmit_type identical for all pings in a channel + first_ping_transmit_type = transmit_type.isel(ping_time=0) if waveform_mode == "BB": # check BB waveform_mode, BB must always have complex data, can have 2 beam groups # when echodata contains CW power and BB complex samples - if np.all(echodata["Sonar/Beam_group1"]["transmit_type"] == "CW"): - raise ValueError("waveform_mode='BB', but broadband data not found!") - elif "backscatter_i" not in echodata["Sonar/Beam_group1"].variables: + if np.all(first_ping_transmit_type == "CW"): raise ValueError("waveform_mode='BB', but complex data does not exist!") elif echodata["Sonar/Beam_group2"] is not None: power_ed_group = "Sonar/Beam_group2" @@ -139,9 +140,7 @@ def _retrieve_correct_beam_group_EK80( # 3) power samples are in Sonar/Beam_group2 if two beam groups exist # Raise error if waveform_mode="CW" but CW data does not exist (not a single ping is CW) - if encode_mode == "complex" and np.all( - echodata["Sonar/Beam_group1"]["transmit_type"] != "CW" - ): + if encode_mode == "complex" and np.all(transmit_type != "CW"): raise ValueError("waveform_mode='CW', but all data are broadband (BB)!") if echodata["Sonar/Beam_group2"] is None: From 5d8ed89da97408415b5d2079066b7535488b0742 Mon Sep 17 00:00:00 2001 From: Emilio Mayorga Date: Fri, 4 Aug 2023 08:47:01 -0700 Subject: [PATCH 18/20] Update echopype/echodata/simrad.py Co-authored-by: Wu-Jung Lee --- echopype/echodata/simrad.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/echopype/echodata/simrad.py b/echopype/echodata/simrad.py index fb6693b24..082e5f2a7 100644 --- a/echopype/echodata/simrad.py +++ b/echopype/echodata/simrad.py @@ -140,7 +140,8 @@ def _retrieve_correct_beam_group_EK80( # 3) power samples are in Sonar/Beam_group2 if two beam groups exist # Raise error if waveform_mode="CW" but CW data does not exist (not a single ping is CW) - if encode_mode == "complex" and np.all(transmit_type != "CW"): + # TODO: change when allowing within-channel CW-BB switch + if encode_mode == "complex" and np.all(first_ping_transmit_type != "CW"): raise ValueError("waveform_mode='CW', but all data are broadband (BB)!") if echodata["Sonar/Beam_group2"] is None: From db22b400a364c08ebc215d98fd4cb126c69eddc2 Mon Sep 17 00:00:00 2001 From: Emilio Mayorga Date: Fri, 4 Aug 2023 08:47:24 -0700 Subject: [PATCH 19/20] Update echopype/echodata/simrad.py Co-authored-by: Wu-Jung Lee --- echopype/echodata/simrad.py | 1 + 1 file changed, 1 insertion(+) diff --git a/echopype/echodata/simrad.py b/echopype/echodata/simrad.py index 082e5f2a7..66cc37efc 100644 --- a/echopype/echodata/simrad.py +++ b/echopype/echodata/simrad.py @@ -122,6 +122,7 @@ def _retrieve_correct_beam_group_EK80( transmit_type = echodata["Sonar/Beam_group1"]["transmit_type"] # assume transmit_type identical for all pings in a channel + # TODO: change when allowing within-channel CW-BB switch first_ping_transmit_type = transmit_type.isel(ping_time=0) if waveform_mode == "BB": # check BB waveform_mode, BB must always have complex data, can have 2 beam groups From 0f81fabd723a92d5dd32f541fb082f5aa153f2aa Mon Sep 17 00:00:00 2001 From: Emilio Mayorga Date: Fri, 4 Aug 2023 11:02:12 -0700 Subject: [PATCH 20/20] Change transmit_type FM string to LFM, per convention, and add more detailed flag_meanings strings, largely from the convention --- echopype/calibrate/calibrate_ek.py | 2 +- echopype/convert/set_groups_azfp.py | 2 +- echopype/convert/set_groups_ek60.py | 2 +- echopype/convert/set_groups_ek80.py | 13 ++++++++----- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/echopype/calibrate/calibrate_ek.py b/echopype/calibrate/calibrate_ek.py index 591aa4fdb..351831a77 100644 --- a/echopype/calibrate/calibrate_ek.py +++ b/echopype/calibrate/calibrate_ek.py @@ -318,7 +318,7 @@ def _get_chan_dict(beam: xr.Dataset) -> Dict: beam["transmit_type"].isel(ping_time=0).drop_vars("ping_time") ) # noqa return { - # For BB: Keep only non-CW channels (FM or FMD) based on transmit_type + # For BB: Keep only non-CW channels (LFM or FMD) based on transmit_type "BB": first_ping_transmit_type.where( first_ping_transmit_type != "CW", drop=True ).channel, # noqa diff --git a/echopype/convert/set_groups_azfp.py b/echopype/convert/set_groups_azfp.py index 9cd3c5759..7af189f34 100644 --- a/echopype/convert/set_groups_azfp.py +++ b/echopype/convert/set_groups_azfp.py @@ -425,7 +425,7 @@ def set_beam(self) -> List[xr.Dataset]: "long_name": "Type of transmitted pulse", "flag_values": ["CW"], "flag_meanings": [ - "Continuous Wave", + "Continuous Wave – a pulse nominally of one frequency", ], }, ), diff --git a/echopype/convert/set_groups_ek60.py b/echopype/convert/set_groups_ek60.py index 28d071408..8a0d23846 100644 --- a/echopype/convert/set_groups_ek60.py +++ b/echopype/convert/set_groups_ek60.py @@ -602,7 +602,7 @@ def set_beam(self) -> List[xr.Dataset]: "long_name": "Type of transmitted pulse", "flag_values": ["CW"], "flag_meanings": [ - "Continuous Wave", + "Continuous Wave – a pulse nominally of one frequency", ], }, ), diff --git a/echopype/convert/set_groups_ek80.py b/echopype/convert/set_groups_ek80.py index 3e22fad2e..8221b701f 100644 --- a/echopype/convert/set_groups_ek80.py +++ b/echopype/convert/set_groups_ek80.py @@ -898,7 +898,7 @@ def _assemble_ds_common(self, ch, range_sample_size): ) def pulse_form_map(pulse_form): - str_map = np.array(["CW", "FM", "", "", "", "FMD"]) + str_map = np.array(["CW", "LFM", "", "", "", "FMD"]) return str_map[pulse_form] ds_common = xr.Dataset( @@ -949,11 +949,14 @@ def pulse_form_map(pulse_form): pulse_form_map(np.array(self.parser_obj.ping_data_dict["pulse_form"][ch])), { "long_name": "Type of transmitted pulse", - "flag_values": ["CW", "FM", "FMD"], + "flag_values": ["CW", "LFM", "FMD"], "flag_meanings": [ - "Continuous Wave", - "Frequency Modulated", - "Frequency Modulated D", + "Continuous Wave – a pulse nominally of one frequency", + "Linear Frequency Modulation – a pulse which varies from " + "transmit_frequency_start to transmit_frequency_stop in a linear " + "manner over the nominal pulse duration (transmit_duration_nominal)", + "Frequency Modulated 'D' - An EK80-specific FM type that is not " + "clearly described", ], }, ),