From 4c293c783a25f7e07880474d81b2b0873563e06a Mon Sep 17 00:00:00 2001 From: Cody Baker Date: Sat, 18 May 2024 21:50:43 -0400 Subject: [PATCH 1/3] debug multiparams --- .../benchmarks/time_remote_slicing.py | 16 +++++----- src/nwb_benchmarks/setup/_reduce_results.py | 29 +++++++++++-------- 2 files changed, 25 insertions(+), 20 deletions(-) diff --git a/src/nwb_benchmarks/benchmarks/time_remote_slicing.py b/src/nwb_benchmarks/benchmarks/time_remote_slicing.py index 15b44b5..ee7cf98 100644 --- a/src/nwb_benchmarks/benchmarks/time_remote_slicing.py +++ b/src/nwb_benchmarks/benchmarks/time_remote_slicing.py @@ -33,14 +33,14 @@ object_name="TwoPhotonSeries", slice_range=(slice(0, 3), slice(0, 796), slice(0, 512)), ), - # IcephysTestCase=dict( - # s3_url=get_s3_url( - # dandiset_id="000717", - # dandi_path="sub-1214579789_ses-1214621812_icephys/sub-1214579789_ses-1214621812_icephys.nwb", - # ), - # object_name="data_00002_AD0", - # slice_range=(slice(0, 30_000),), - # ), + IcephysTestCase=dict( + s3_url=get_s3_url( + dandiset_id="000717", + dandi_path="sub-1214579789_ses-1214621812_icephys/sub-1214579789_ses-1214621812_icephys.nwb", + ), + object_name="data_00002_AD0", + slice_range=(slice(0, 30_000),), + ), ) zarr_parameter_cases = dict( diff --git a/src/nwb_benchmarks/setup/_reduce_results.py b/src/nwb_benchmarks/setup/_reduce_results.py index b52677b..71c8629 100644 --- a/src/nwb_benchmarks/setup/_reduce_results.py +++ b/src/nwb_benchmarks/setup/_reduce_results.py @@ -8,6 +8,7 @@ import shutil import subprocess import sys +import warnings from typing import Dict, List @@ -58,19 +59,23 @@ def reduce_results(raw_results_file_path: pathlib.Path, raw_environment_info_fil # Skipped results in JSON are writen as `null` and read back into Python as `None` non_skipped_results = [result for result in raw_results_list[11] if result is not None] - assert len(serialized_flattened_joint_params) == len(non_skipped_results), ( - f"Length mismatch between flattened joint parameters ({serialized_flattened_joint_params} and result " - f"samples ({len(non_skipped_results)}) in test case '{test_case}'! " - "Please raise an issue and share your intermediate results file." - ) - reduced_results.update( - { - test_case: { - joint_params: raw_result - for joint_params, raw_result in zip(serialized_flattened_joint_params, non_skipped_results) + if len(serialized_flattened_joint_params) != len(non_skipped_results): + message = ( + f"In intermediate results for test case {test_case}: \n" + f"\tLength mismatch between flattened joint parameters ({len(serialized_flattened_joint_params)}) and " + f"result samples ({len(non_skipped_results)})!\n\n" + "Please raise an issue and share your intermediate results file." + ) + warnings.warn(message=message) + else: + reduced_results.update( + { + test_case: { + joint_params: raw_result + for joint_params, raw_result in zip(serialized_flattened_joint_params, non_skipped_results) + } } - } - ) + ) if len(reduced_results) == 0: raise ValueError( From 2b359bffe2a1c2061d2ae13f0a85f3422b1d04ca Mon Sep 17 00:00:00 2001 From: Cody Baker Date: Sat, 18 May 2024 22:41:54 -0400 Subject: [PATCH 2/3] some results --- ...f3abc58d3f9487dc3b25977d808fc1536e53f.json | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 results/results_timestamp-2024-05-18-21-50-43_machine-Ackermann_environment-91bf3abc58d3f9487dc3b25977d808fc1536e53f.json diff --git a/results/results_timestamp-2024-05-18-21-50-43_machine-Ackermann_environment-91bf3abc58d3f9487dc3b25977d808fc1536e53f.json b/results/results_timestamp-2024-05-18-21-50-43_machine-Ackermann_environment-91bf3abc58d3f9487dc3b25977d808fc1536e53f.json new file mode 100644 index 0000000..6b2a01f --- /dev/null +++ b/results/results_timestamp-2024-05-18-21-50-43_machine-Ackermann_environment-91bf3abc58d3f9487dc3b25977d808fc1536e53f.json @@ -0,0 +1,57 @@ +{ + "version": 2, + "timestamp": "2024-05-18-21-50-43", + "commit_hash": "4c293c783a25f7e07880474d81b2b0873563e06a", + "environment_hash": "91bf3abc58d3f9487dc3b25977d808fc1536e53f", + "machine_hash": "Ackermann", + "results": { + "time_remote_slicing.FsspecNoCacheContinuousSliceBenchmark.time_slice": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": [ + 0.577229900052771, + 0.5970238998997957, + 0.5664639000315219 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/38c/c24/38cc240b-77c5-418a-9040-a7f582ff6541'\", \"'TwoPhotonSeries'\", '(slice(0, 3, None), slice(0, 796, None), slice(0, 512, None))')": [ + 1.0336655001156032, + 1.2907407998573035, + 0.9944616998545825 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/c98/3a4/c983a4e1-097a-402c-bda8-e6a41cb7e24a'\", \"'data_00002_AD0'\", '(slice(0, 30000, None),)')": [ + 0.17991229984909296, + 0.2016158001497388, + 0.18414429994300008 + ] + }, + "time_remote_slicing.Ros3ContinuousSliceBenchmark.time_slice": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": [ + 0.0035342001356184483, + 0.003450800199061632, + 0.0036127998027950525 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/38c/c24/38cc240b-77c5-418a-9040-a7f582ff6541'\", \"'TwoPhotonSeries'\", '(slice(0, 3, None), slice(0, 796, None), slice(0, 512, None))')": [ + 0.0007092999294400215, + 0.0007117001805454493, + 0.000698799965903163 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/c98/3a4/c983a4e1-097a-402c-bda8-e6a41cb7e24a'\", \"'data_00002_AD0'\", '(slice(0, 30000, None),)')": [ + 6.3811555066292796e-06, + 6.141684663765631e-06, + 6.1631210492300295e-06 + ] + }, + "time_remote_slicing.ZarrContinuousSliceBenchmark.time_slice": { + "(\"'s3://aind-open-data/ecephys_625749_2022-08-03_15-15-06_nwb_2023-05-16_16-34-55/ecephys_625749_2022-08-03_15-15-06_nwb/ecephys_625749_2022-08-03_15-15-06_experiment1_recording1.nwb.zarr/'\", \"'ElectricalSeriesProbe A-LFP'\", '(slice(0, 30000, None), slice(0, 384, None))')": [ + 1.2064452001359314, + 1.45879239984788, + 1.137998699909076 + ] + }, + "time_remote_slicing.ZarrForceNoConsolidatedContinuousSliceBenchmark.time_slice": { + "(\"'s3://aind-open-data/ecephys_625749_2022-08-03_15-15-06_nwb_2023-05-16_16-34-55/ecephys_625749_2022-08-03_15-15-06_nwb/ecephys_625749_2022-08-03_15-15-06_experiment1_recording1.nwb.zarr/'\", \"'ElectricalSeriesProbe A-LFP'\", '(slice(0, 30000, None), slice(0, 384, None))')": [ + 1.0744982999749482, + 1.1373234000056982, + 1.347178699914366 + ] + } + } +} From 077a763260080db8d1d054194c3c86900b48050b Mon Sep 17 00:00:00 2001 From: Cody Baker Date: Sun, 19 May 2024 00:12:52 -0400 Subject: [PATCH 3/3] almost full results from slice timings --- ...f3abc58d3f9487dc3b25977d808fc1536e53f.json | 108 ++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 results/results_timestamp-2024-05-18-22-41-54_machine-Ackermann_environment-91bf3abc58d3f9487dc3b25977d808fc1536e53f.json diff --git a/results/results_timestamp-2024-05-18-22-41-54_machine-Ackermann_environment-91bf3abc58d3f9487dc3b25977d808fc1536e53f.json b/results/results_timestamp-2024-05-18-22-41-54_machine-Ackermann_environment-91bf3abc58d3f9487dc3b25977d808fc1536e53f.json new file mode 100644 index 0000000..44940e4 --- /dev/null +++ b/results/results_timestamp-2024-05-18-22-41-54_machine-Ackermann_environment-91bf3abc58d3f9487dc3b25977d808fc1536e53f.json @@ -0,0 +1,108 @@ +{ + "version": 2, + "timestamp": "2024-05-18-22-41-54", + "commit_hash": "2b359bffe2a1c2061d2ae13f0a85f3422b1d04ca", + "environment_hash": "91bf3abc58d3f9487dc3b25977d808fc1536e53f", + "machine_hash": "Ackermann", + "results": { + "time_remote_slicing.FsspecNoCacheContinuousSliceBenchmark.time_slice": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": [ + 0.5770294000394642, + 0.5558074999134988, + 0.5851527999620885 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/38c/c24/38cc240b-77c5-418a-9040-a7f582ff6541'\", \"'TwoPhotonSeries'\", '(slice(0, 3, None), slice(0, 796, None), slice(0, 512, None))')": [ + 1.0202222999650985, + 1.0687915000598878, + 1.1272402999456972 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/c98/3a4/c983a4e1-097a-402c-bda8-e6a41cb7e24a'\", \"'data_00002_AD0'\", '(slice(0, 30000, None),)')": [ + 0.1669504998717457, + 0.15758339990861714, + 0.23897469998337328 + ] + }, + "time_remote_slicing.FsspecWithCacheContinuousSliceBenchmark.time_slice": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": [ + 0.46691940003074706, + 0.434342100052163, + 0.42370989988557994 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/38c/c24/38cc240b-77c5-418a-9040-a7f582ff6541'\", \"'TwoPhotonSeries'\", '(slice(0, 3, None), slice(0, 796, None), slice(0, 512, None))')": [ + 0.29375869990326464, + 0.26918979990296066, + 0.286825800081715 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/c98/3a4/c983a4e1-097a-402c-bda8-e6a41cb7e24a'\", \"'data_00002_AD0'\", '(slice(0, 30000, None),)')": [ + 6.63219959654271e-05, + 5.873775452703179e-05, + 6.128844388626707e-05 + ] + }, + "time_remote_slicing.RemfileNoCacheContinuousSliceBenchmark.time_slice": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": [ + 0.5421017999760807, + 0.5975515998434275, + 0.6133785999845713 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/38c/c24/38cc240b-77c5-418a-9040-a7f582ff6541'\", \"'TwoPhotonSeries'\", '(slice(0, 3, None), slice(0, 796, None), slice(0, 512, None))')": [ + 0.5068071000277996, + 0.5669373997952789, + 0.5368534000590444 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/c98/3a4/c983a4e1-097a-402c-bda8-e6a41cb7e24a'\", \"'data_00002_AD0'\", '(slice(0, 30000, None),)')": [ + 6.4188481480264605e-06, + 6.4300173820972165e-06, + 6.257184445771317e-06 + ] + }, + "time_remote_slicing.RemfileWithCacheContinuousSliceBenchmark.time_slice": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": [ + 0.7116888999007642, + 0.7746200999245048, + 0.7001597001217306 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/38c/c24/38cc240b-77c5-418a-9040-a7f582ff6541'\", \"'TwoPhotonSeries'\", '(slice(0, 3, None), slice(0, 796, None), slice(0, 512, None))')": [ + 0.6674359999597073, + 0.5512343000154942, + 0.5494779001455754 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/c98/3a4/c983a4e1-097a-402c-bda8-e6a41cb7e24a'\", \"'data_00002_AD0'\", '(slice(0, 30000, None),)')": [ + 6.591881237390126e-06, + 6.737569993630903e-06, + 6.343617024732397e-06 + ] + }, + "time_remote_slicing.Ros3ContinuousSliceBenchmark.time_slice": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": [ + 0.0033102999441325665, + 0.0034116001334041357, + 0.0033524997998028994 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/38c/c24/38cc240b-77c5-418a-9040-a7f582ff6541'\", \"'TwoPhotonSeries'\", '(slice(0, 3, None), slice(0, 796, None), slice(0, 512, None))')": [ + 0.0008107998874038458, + 0.0007931999862194061, + 0.0007380000315606594 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/c98/3a4/c983a4e1-097a-402c-bda8-e6a41cb7e24a'\", \"'data_00002_AD0'\", '(slice(0, 30000, None),)')": [ + 6.302597341961204e-06, + 5.955957081368722e-06, + 6.150705781925674e-06 + ] + }, + "time_remote_slicing.ZarrContinuousSliceBenchmark.time_slice": { + "(\"'s3://aind-open-data/ecephys_625749_2022-08-03_15-15-06_nwb_2023-05-16_16-34-55/ecephys_625749_2022-08-03_15-15-06_nwb/ecephys_625749_2022-08-03_15-15-06_experiment1_recording1.nwb.zarr/'\", \"'ElectricalSeriesProbe A-LFP'\", '(slice(0, 30000, None), slice(0, 384, None))')": [ + 1.1419850999955088, + 1.1669653998687863, + 1.228188700042665 + ] + }, + "time_remote_slicing.ZarrForceNoConsolidatedContinuousSliceBenchmark.time_slice": { + "(\"'s3://aind-open-data/ecephys_625749_2022-08-03_15-15-06_nwb_2023-05-16_16-34-55/ecephys_625749_2022-08-03_15-15-06_nwb/ecephys_625749_2022-08-03_15-15-06_experiment1_recording1.nwb.zarr/'\", \"'ElectricalSeriesProbe A-LFP'\", '(slice(0, 30000, None), slice(0, 384, None))')": [ + 1.0642591000068933, + 1.097299600020051, + 1.493450199952349 + ] + } + } +}