From 4dac76f27563c44e20f2fae6cdaaca79b34ace84 Mon Sep 17 00:00:00 2001 From: mingxueg-nv Date: Thu, 22 Aug 2024 19:08:39 +0800 Subject: [PATCH] add check apps properties (#619) Fixes # . ### Description Add check different apps properties in monai ci `verify_bundle()`. ### Status **Work in progress** ### Please ensure all the checkboxes: - [x] Codeformat tests passed locally by running `./runtests.sh --codeformat`. - [ ] In-line docstrings updated. - [x] Update `version` and `changelog` in `metadata.json` if changing an existing bundle. - [ ] Please ensure the naming rules in config files meet our requirements (please refer to: `CONTRIBUTING.md`). - [ ] Ensure versions of packages such as `monai`, `pytorch` and `numpy` are correct in `metadata.json`. - [ ] Descriptions should be consistent with the content, such as `eval_metrics` of the provided weights and TorchScript modules. - [ ] Files larger than 25MB are excluded and replaced by providing download links in `large_file.yml`. - [ ] Avoid using path that contains personal information within config files (such as use `/home/your_name/` for `"bundle_root"`). --------- Signed-off-by: Yiheng Wang Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Yiheng Wang Co-authored-by: Yiheng Wang <68361391+yiheng-wang-nv@users.noreply.github.com> --- ci/bundle_custom_data.py | 6 ++++ ci/verify_bundle.py | 60 +++++++++++++++++++++++++++------------- 2 files changed, 47 insertions(+), 19 deletions(-) diff --git a/ci/bundle_custom_data.py b/ci/bundle_custom_data.py index 3f5aed33..8d391437 100644 --- a/ci/bundle_custom_data.py +++ b/ci/bundle_custom_data.py @@ -41,6 +41,12 @@ "maisi_ct_generative", ] +# This list is used for our CI tests to determine whether a bundle needs to be tested after downloading +# the large files. +# If a bundle contains large files that are too lage (like > 10GB), please add the bundle name into the list. +# For bundles in this list, related tests will be skipped in Github Actions, but they will still be tested in blossom CI. +exclude_download_large_file_list = ["maisi_ct_generative"] + # This dict is used for our CI tests to install required dependencies that cannot be installed by `pip install` directly. # If a bundle has this kind of dependencies, please add the bundle name (key), and the path of the install script (value) # into the dict. diff --git a/ci/verify_bundle.py b/ci/verify_bundle.py index 3780ff25..eae5b27e 100644 --- a/ci/verify_bundle.py +++ b/ci/verify_bundle.py @@ -11,10 +11,12 @@ import argparse import os +import shutil import sys import torch from bundle_custom_data import ( + exclude_download_large_file_list, exclude_verify_preferred_files_list, exclude_verify_shape_list, exclude_verify_torchscript_list, @@ -86,7 +88,7 @@ def _find_license_file(bundle_path: str): return None -def verify_bundle_directory(models_path: str, bundle_name: str): +def verify_bundle_directory(models_path: str, bundle_name: str, mode: str): """ According to [MONAI Bundle Specification](https://docs.monai.io/en/latest/mb_specification.html), as well as the requirements of model zoo, some files are necessary with the bundle. For example: @@ -100,12 +102,15 @@ def verify_bundle_directory(models_path: str, bundle_name: str): bundle_path = os.path.join(models_path, bundle_name) # download large files (if exist) first - large_file_name = _find_bundle_file(bundle_path, "large_files") - if large_file_name is not None: - try: - download_large_files(bundle_path=bundle_path, large_file_name=large_file_name) - except Exception as e: - raise ValueError(f"Download large files in {bundle_path} error.") from e + if bundle_name in exclude_download_large_file_list and mode == "min": + print(f"skip downloading large files for bundle: {bundle_name}.") + else: + large_file_name = _find_bundle_file(bundle_path, "large_files") + if large_file_name is not None: + try: + download_large_files(bundle_path=bundle_path, large_file_name=large_file_name) + except Exception as e: + raise ValueError(f"Download large files in {bundle_path} error.") from e # verify necessary files are included for file in necessary_files_list: @@ -218,21 +223,37 @@ def get_app_properties(app: str, version: str): This function is used to get the properties file of the app. """ - if app in ["monai-deploy", "monai-label", "nvflare"]: - # TODO: so far these apps use default properties, need to grab the properties from the app instead. + # dir structure: ./app/ + cur_dir = os.path.dirname(os.path.abspath(__file__)) + if "ci" in cur_dir: + cur_dir = os.path.dirname(cur_dir) + for root, _dirs, files in os.walk(os.path.join(cur_dir, app)): + if "bundle_properties.py" in files and os.path.isfile(os.path.join(root, "bundle_properties.py")): + print(root) + return os.path.join(root, "bundle_properties.py") + else: return None - return None def check_properties(**kwargs): """ This function is used to check the properties of the workflow. - """ + app_properties_path = kwargs.get("properties_path", "") + kwargs.pop("properties_path", None) workflow = create_workflow(**kwargs) - check_result = workflow.check_properties() - if check_result is not None and len(check_result) > 0: - raise ValueError(f"check properties for workflow failed: {check_result}") + if app_properties_path is not None and os.path.isfile(app_properties_path): + shutil.copy(app_properties_path, "ci/bundle_properties.py") + from bundle_properties import InferProperties, MetaProperties + + workflow.properties = {**MetaProperties, **InferProperties} + check_result = workflow.check_properties() + if check_result is not None and len(check_result) > 0: + raise ValueError( + f"check properties for workflow failed: {check_result}, app_properties_path: {app_properties_path}" + ) + else: + print(f"check properties for workflow successfully {check_result}.") def verify_bundle_properties(model_path: str, bundle: str): @@ -261,16 +282,17 @@ def verify_bundle_properties(model_path: str, bundle: str): if "supported_apps" in metadata: supported_apps = metadata["supported_apps"] all_properties = [] + print("vista3d sopperted apps: ", supported_apps) for app, version in supported_apps.items(): - if app in ["vista3d-nim", "maisi-nim"]: - # skip check - continue properties_path = get_app_properties(app, version) - all_properties.append(properties_path) + if properties_path is not None: + all_properties.append(properties_path) all_properties = list(set(all_properties)) + print("all properties: ", all_properties) for properties_path in all_properties: check_property_args["properties_path"] = properties_path check_properties(**check_property_args) + print("successfully checked properties.") else: check_properties(**check_property_args) @@ -280,7 +302,7 @@ def verify(bundle, models_path="models", mode="full"): # add bundle path to ensure custom code can be used sys.path = [os.path.join(models_path, bundle)] + sys.path # verify bundle directory - verify_bundle_directory(models_path, bundle) + verify_bundle_directory(models_path, bundle, mode) print("directory is verified correctly.") if mode != "regular": # verify version, changelog