From 5a5daedc20671cd0ae6a50e91c6e8bc50776761b Mon Sep 17 00:00:00 2001 From: swsuggs <15131284+swsuggs@users.noreply.github.com> Date: Tue, 6 Jun 2023 13:19:46 -0400 Subject: [PATCH 01/14] clarify local repo path in config documentation (#1944) --- docs/configuration_files.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration_files.md b/docs/configuration_files.md index 48940b595..c9d3b8363 100644 --- a/docs/configuration_files.md +++ b/docs/configuration_files.md @@ -70,7 +70,7 @@ All configuration files are verified against the jsonschema definition at run ti external_github_repo: [String or null or Object] External github repository(s) to download and place on PYTHONPATH within container external_github_repo_pythonpath: [String or null or Object] Relative path(s) in the repo directory to add to PYTHONPATH within container gpus: [String]: Which GPUs should the docker container have access to. "all" or comma sperated list (e.g. "1,3") - local_repo_path: [String or null or Object] Local github repository path(s) to place on PYTHONPATH within container + local_repo_path: [String or null or Object] Local github repository path(s) to place on PYTHONPATH within container (relative to the "local_git_dir" variable specified in ~/.armory/config.json) output_dir: [Optional String]: Add an optional output directory prefix to the default output directory name. output_filename: [Optional String]: Optionally change the output filename prefix (from default of scenario name) use_gpu: [Boolean]: Boolean to run container as nvidia-docker with GPU access From d07a56219d327a75bc0d6eca2005b4bc7bf2d270 Mon Sep 17 00:00:00 2001 From: Christopher Woodall Date: Wed, 14 Jun 2023 06:53:33 -0500 Subject: [PATCH 02/14] Update utils.py (#1947) --- armory/data/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/armory/data/utils.py b/armory/data/utils.py index 963771560..0ee5ad7a6 100644 --- a/armory/data/utils.py +++ b/armory/data/utils.py @@ -248,7 +248,7 @@ def move_merge(source, dest): def download_verify_dataset_cache(dataset_dir, checksum_file, name): found_checksum_flag = False - log.info("Attempting download_verigy_dataset_cache with dataset_dir") + log.info("Attempting download_verify_dataset_cache with dataset_dir") for checksum_dir in CHECKSUMS_DIRS: checksum_file_full_path = os.path.join(checksum_dir, checksum_file) if os.path.exists(checksum_file_full_path): From 97e5138181aaecec862f188723ec1f38de3ce711 Mon Sep 17 00:00:00 2001 From: Sterling Suggs Date: Mon, 19 Jun 2023 09:48:15 -0400 Subject: [PATCH 03/14] results for od poisoning with dev triggers --- .../object_detection_poisoning.md | 165 ++++++++++++++++++ 1 file changed, 165 insertions(+) create mode 100644 docs/baseline_results/object_detection_poisoning.md diff --git a/docs/baseline_results/object_detection_poisoning.md b/docs/baseline_results/object_detection_poisoning.md new file mode 100644 index 000000000..1143ca829 --- /dev/null +++ b/docs/baseline_results/object_detection_poisoning.md @@ -0,0 +1,165 @@ +# Global Misclassification Attack + +## Globe trigger + +### Undefended + +| Poison Percentage | Benign mAP | Benign AP target | Adv mAP - Clean labels | Adv AP target - Clean labels | Adv mAP - Adv labels | Adv AP target - Adv labels | Attack success rate | +| ------- | ------- | ------- | ------- | ------- | ------- | ------- | ------- | +| 00 | 0.527 | 0.460 | - | - | - | - | - | +| 01 | 0.499 | 0.467 | 0.500 | 0.470 | 0.081 | 0.243 | 0.016 | +| 05 | 0.514 | 0.493 | 0.169 | 0.313 | 0.149 | 0.447 | 0.620 | +| 10 | 0.487 | 0.463 | 0.173 | 0.323 | 0.152 | 0.457 | 0.635 | +| 20 | 0.510 | 0.467 | 0.166 | 0.317 | 0.164 | 0.493 | 0.701 | +| 30 | 0.472 | 0.453 | 0.156 | 0.327 | 0.162 | 0.487 | 0.688 | + + +### Random Filter + +| Poison Percentage | Benign mAP | Benign AP target | Adv mAP - Clean labels | Adv AP target - Clean labels | Adv mAP - Adv labels | Adv AP target - Adv labels | Attack success rate | +| ------- | ------- | ------- | ------- | ------- | ------- | ------- | ------- | +| 00 | 0.432 | 0.403 | - | - | - | - | - | +| 01 | 0.448 | 0.430 | 0.447 | 0.427 | 0.074 | 0.223 | 0.014 | +| 05 | 0.441 | 0.430 | 0.221 | 0.340 | 0.118 | 0.353 | 0.427 | +| 10 | 0.448 | 0.450 | 0.160 | 0.300 | 0.144 | 0.433 | 0.587 | +| 20 | 0.460 | 0.430 | 0.163 | 0.297 | 0.149 | 0.447 | 0.637 | +| 30 | 0.424 | 0.423 | 0.126 | 0.277 | 0.152 | 0.457 | 0.672 | + + +### Perfect Filter + +| Poison Percentage | Benign mAP | Benign AP target | Adv mAP - Clean labels | Adv AP target - Clean labels | Adv mAP - Adv labels | Adv AP target - Adv labels | Attack success rate | +| ------- | ------- | ------- | ------- | ------- | ------- | ------- | ------- | +| 00 | 0.518 | 0.460 | - | - | - | - | - | +| 01 | 0.549 | 0.473 | 0.549 | 0.473 | 0.082 | 0.247 | 0.024 | +| 05 | 0.538 | 0.473 | 0.539 | 0.473 | 0.081 | 0.243 | 0.016 | +| 10 | 0.516 | 0.457 | 0.517 | 0.457 | 0.079 | 0.237 | 0.020 | +| 20 | 0.480 | 0.440 | 0.480 | 0.437 | 0.074 | 0.223 | 0.014 | +| 30 | 0.481 | 0.423 | 0.480 | 0.427 | 0.073 | 0.220 | 0.012 | + + + + +## Baby-on-board trigger + +### Undefended + +| Poison Percentage | Benign mAP | Benign AP target | Adv mAP - Clean labels | Adv AP target - Clean labels | Adv mAP - Adv labels | Adv AP target - Adv labels | Attack success rate | +| ------- | ------- | ------- | ------- | ------- | ------- | ------- | ------- | +| 00 | 0.518 | 0.457 | - | - | - | - | - | +| 01 | 0.508 | 0.487 | 0.507 | 0.487 | 0.080 | 0.240 | 0.014 | +| 05 | 0.488 | 0.473 | 0.379 | 0.433 | 0.106 | 0.317 | 0.219 | +| 10 | 0.502 | 0.457 | 0.157 | 0.290 | 0.149 | 0.447 | 0.623 | +| 20 | 0.524 | 0.483 | 0.173 | 0.343 | 0.173 | 0.520 | 0.704 | +| 30 | 0.484 | 0.460 | 0.140 | 0.270 | 0.142 | 0.397 | 0.566 | + + +### Random Filter + +| Poison Percentage | Benign mAP | Benign AP target | Adv mAP - Clean labels | Adv AP target - Clean labels | Adv mAP - Adv labels | Adv AP target - Adv labels | Attack success rate | +| ------- | ------- | ------- | ------- | ------- | ------- | ------- | ------- | +| 00 | 0.454 | 0.430 | - | - | - | - | - | +| 01 | 0.463 | 0.417 | 0.464 | 0.417 | 0.076 | 0.227 | 0.024 | +| 05 | 0.427 | 0.417 | 0.350 | 0.390 | 0.089 | 0.267 | 0.186 | +| 10 | 0.462 | 0.450 | 0.156 | 0.277 | 0.146 | 0.437 | 0.592 | +| 20 | 0.461 | 0.443 | 0.146 | 0.273 | 0.160 | 0.480 | 0.690 | +| 30 | 0.406 | 0.403 | 0.121 | 0.283 | 0.159 | 0.420 | 0.628 | + + +### Perfect Filter + +| Poison Percentage | Benign mAP | Benign AP target | Adv mAP - Clean labels | Adv AP target - Clean labels | Adv mAP - Adv labels | Adv AP target - Adv labels | Attack success rate | +| ------- | ------- | ------- | ------- | ------- | ------- | ------- | ------- | +| 00 | 0.519 | 0.463 | - | - | - | - | - | +| 01 | 0.532 | 0.467 | 0.532 | 0.467 | 0.082 | 0.247 | 0.026 | +| 05 | 0.521 | 0.460 | 0.520 | 0.460 | 0.079 | 0.237 | 0.017 | +| 10 | 0.529 | 0.457 | 0.530 | 0.457 | 0.080 | 0.240 | 0.016 | +| 20 | 0.518 | 0.447 | 0.519 | 0.450 | 0.079 | 0.237 | 0.014 | +| 30 | 0.493 | 0.447 | 0.494 | 0.447 | 0.077 | 0.230 | 0.015 | + + + + + +# Object Generation Attack + +## Globe Trigger + +### Undefended + +| Poison Percentage | Benign mAP | Benign AP target | Adv mAP - Clean labels | Adv AP target - Clean labels | Adv mAP - Adv labels | Adv AP target - Adv labels | Attack success rate | +| ------- | ------- | ------- | ------- | ------- | ------- | ------- | ------- | +| 00 | 0.529 | 0.463 | - | - | - | - | - | +| 01 | 0.524 | 0.493 | - | - | 0.499 | 0.457 | 0.709 | +| 05 | 0.514 | 0.473 | - | - | 0.519 | 0.533 | 0.915 | +| 10 | 0.534 | 0.487 | - | - | 0.533 | 0.560 | 0.931 | +| 20 | 0.529 | 0.477 | - | - | 0.542 | 0.553 | 0.925 | +| 30 | 0.534 | 0.477 | - | - | 0.531 | 0.547 | 0.925 | + + +### Random Filter + +| Poison Percentage | Benign mAP | Benign AP target | Adv mAP - Clean labels | Adv AP target - Clean labels | Adv mAP - Adv labels | Adv AP target - Adv labels | Attack success rate | +| ------- | ------- | ------- | ------- | ------- | ------- | ------- | ------- | +| 00 | 0.459 | 0.430 | - | - | - | - | - | +| 01 | 0.470 | 0.443 | - | - | 0.426 | 0.417 | 0.666 | +| 05 | 0.487 | 0.447 | - | - | 0.466 | 0.513 | 0.933 | +| 10 | 0.440 | 0.423 | - | - | 0.444 | 0.520 | 0.919 | +| 20 | 0.430 | 0.397 | - | - | 0.450 | 0.490 | 0.942 | +| 30 | 0.507 | 0.453 | - | - | 0.510 | 0.530 | 0.929 | + + +### Perfect Filter + +| Poison Percentage | Benign mAP | Benign AP target | Adv mAP - Clean labels | Adv AP target - Clean labels | Adv mAP - Adv labels | Adv AP target - Adv labels | Attack success rate | +| ------- | ------- | ------- | ------- | ------- | ------- | ------- | ------- | +| 00 | 0.530 | 0.470 | - | - | - | - | - | +| 01 | 0.526 | 0.463 | - | - | 0.449 | 0.227 | 0.001 | +| 05 | 0.516 | 0.450 | - | - | 0.437 | 0.220 | 0.002 | +| 10 | 0.520 | 0.453 | - | - | 0.430 | 0.217 | 0.001 | +| 20 | 0.492 | 0.447 | - | - | 0.409 | 0.213 | 0.000 | +| 30 | 0.456 | 0.440 | - | - | 0.379 | 0.213 | 0.000 | + + + + +## Baby-on-board trigger + +### Undefended + +| Poison Percentage | Benign mAP | Benign AP target | Adv mAP - Clean labels | Adv AP target - Clean labels | Adv mAP - Adv labels | Adv AP target - Adv labels | Attack success rate | +| ------- | ------- | ------- | ------- | ------- | ------- | ------- | ------- | +| 00 | 0.506 | 0.463 | - | - | - | - | - | +| 01 | 0.524 | 0.457 | - | - | 0.496 | 0.500 | 0.840 | +| 05 | 0.537 | 0.470 | - | - | 0.526 | 0.547 | 0.930 | +| 10 | 0.527 | 0.467 | - | - | 0.537 | 0.553 | 0.940 | +| 20 | 0.527 | 0.457 | - | - | 0.556 | 0.570 | 0.934 | +| 30 | 0.548 | 0.487 | - | - | 0.559 | 0.573 | 0.935 | + + +### Random Filter + +| Poison Percentage | Benign mAP | Benign AP target | Adv mAP - Clean labels | Adv AP target - Clean labels | Adv mAP - Adv labels | Adv AP target - Adv labels | Attack success rate | +| ------- | ------- | ------- | ------- | ------- | ------- | ------- | ------- | +| 00 | 0.452 | 0.433 | - | - | - | - | - | +| 01 | 0.440 | 0.423 | - | - | 0.413 | 0.427 | 0.856 | +| 05 | 0.456 | 0.437 | - | - | 0.453 | 0.523 | 0.919 | +| 10 | 0.494 | 0.423 | - | - | 0.474 | 0.507 | 0.932 | +| 20 | 0.456 | 0.433 | - | - | 0.462 | 0.507 | 0.939 | +| 30 | 0.499 | 0.457 | - | - | 0.516 | 0.530 | 0.930 | + + +### Perfect Filter + +| Poison Percentage | Benign mAP | Benign AP target | Adv mAP - Clean labels | Adv AP target - Clean labels | Adv mAP - Adv labels | Adv AP target - Adv labels | Attack success rate | +| ------- | ------- | ------- | ------- | ------- | ------- | ------- | ------- | +| 00 | 0.532 | 0.480 | - | - | - | - | - | +| 01 | 0.526 | 0.460 | - | - | 0.444 | 0.223 | 0.000 | +| 05 | 0.529 | 0.460 | - | - | 0.446 | 0.220 | 0.001 | +| 10 | 0.527 | 0.447 | - | - | 0.442 | 0.220 | 0.001 | +| 20 | 0.503 | 0.443 | - | - | 0.418 | 0.213 | 0.000 | +| 30 | 0.474 | 0.437 | - | - | 0.399 | 0.210 | 0.001 | + + + + From add8660d5b6d0a3b8538b121b1317347702fc44c Mon Sep 17 00:00:00 2001 From: Sterling Suggs Date: Mon, 19 Jun 2023 09:50:26 -0400 Subject: [PATCH 04/14] rename results file --- ...tection_poisoning.md => object_detection_poisoning_results.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/baseline_results/{object_detection_poisoning.md => object_detection_poisoning_results.md} (100%) diff --git a/docs/baseline_results/object_detection_poisoning.md b/docs/baseline_results/object_detection_poisoning_results.md similarity index 100% rename from docs/baseline_results/object_detection_poisoning.md rename to docs/baseline_results/object_detection_poisoning_results.md From a8ebb42fc1dcfe77db942604c2fe3af3f665fe3d Mon Sep 17 00:00:00 2001 From: Sterling Suggs Date: Mon, 19 Jun 2023 09:50:44 -0400 Subject: [PATCH 05/14] link results to scenarios doc --- docs/scenarios.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/scenarios.md b/docs/scenarios.md index b7bcd92f2..a0bca67cc 100644 --- a/docs/scenarios.md +++ b/docs/scenarios.md @@ -546,7 +546,7 @@ For a complete overview, see the [poisoning doc](poisoning.md). * [Random Filter](https://github.com/twosixlabs/armory/blob/master/armory/art_experimental/poison_detection/random_filter.py) * [Perfect Filter](https://github.com/twosixlabs/armory/blob/1d6caa9166313c1409edbbc5f089d2bc774b5230/armory/scenarios/poison.py#L233-L235) * **Baseline Evaluations:** - * Minicoco Results to be added + * [MiniCoco results](baseline_results/object_detection_poisoning_results.md) From da23ab026e59491e9afc7c52faf3767e2617ed71 Mon Sep 17 00:00:00 2001 From: Sterling Suggs Date: Mon, 19 Jun 2023 09:59:41 -0400 Subject: [PATCH 06/14] additional descriptions --- .../object_detection_poisoning_results.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/baseline_results/object_detection_poisoning_results.md b/docs/baseline_results/object_detection_poisoning_results.md index 1143ca829..a83e68618 100644 --- a/docs/baseline_results/object_detection_poisoning_results.md +++ b/docs/baseline_results/object_detection_poisoning_results.md @@ -1,3 +1,16 @@ +# Object Detection Poisoning Baseline Evaluation + +Containing results for Global Misclassification and Object Generation attacks. + +**All tables are the mean of 3 runs.** Results obtained with Armory 0.17.2 June 2023. + +Relevant parameters: +``` +"target_class": 1 +"score_threshold": 0.05, +``` +(Source class is N/A for these two attacks) + # Global Misclassification Attack ## Globe trigger @@ -83,6 +96,12 @@ # Object Generation Attack +Generated box parameters: +``` +"bbox_height": 70, +"bbox_width": 50 +``` + ## Globe Trigger ### Undefended From bea8b313745b56b8d009e2bbb1fd25fe039434f6 Mon Sep 17 00:00:00 2001 From: Sterling Suggs Date: Mon, 19 Jun 2023 10:04:08 -0400 Subject: [PATCH 07/14] a few updates to poisoning doc --- docs/poisoning.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/poisoning.md b/docs/poisoning.md index bac3da67e..ee6a67aaf 100644 --- a/docs/poisoning.md +++ b/docs/poisoning.md @@ -1,13 +1,13 @@ # Poisoning -Updated October 2022 +Updated June 2023 Armory supports a handful of specific poisoning threat models and attacks. This document will first describe these, providing enough background for newcomers to get up to speed on what these attacks do. Then, the peculiarities of the poisoning configs will be addressed, including lots of helpful information about Witches' Brew. Finally, we will describe the poisoning-specific metrics. ## Threat Models -There are currently four threat models handled by Armory: dirty-label backdoor, clean-label backdoor, Witches' Brew (clean-label gradient matching), and Sleeper Agent. In a backdoor attack, an adversary adds a small trigger, or backdoor, to a small portion of the train set in order to gain control of the the model at test time. +Armory supports several poisoning threat models: dirty-label backdoor, clean-label backdoor, Witches' Brew and Sleeper Agent (clean-label gradient matching), and Object Detection (dirty-label backdoor). In a backdoor attack, an adversary adds a small trigger, or backdoor, to a small portion of the train set in order to gain control of the the model at test time. The trigger is usually a small (but not imperceptible) image superposed on the data, and the adversary's goal is to force the model to misclassify test images that have the trigger applied. Armory includes several trigger images under `utils/triggers/`. @@ -20,7 +20,7 @@ In a [Dirty-label Backdoor (DLBD) Attack](https://arxiv.org/abs/1708.06733), tra #### Audio -The DLBD attack for audio is similar to that of video. The difference is that instead of the trigger being an image that is placed over the existing image, the trigger is a short audio clip that is mixed with the existing audio. Example configs for speech are [here](../scenario_configs/eval6/poisoning) +The DLBD attack for audio is similar to that of image. The difference is that instead of the trigger being an image that is placed over the existing image, the trigger is a short audio clip that is mixed with the existing audio. Example configs for speech are [here](../scenario_configs/eval6/poisoning) Current triggers include a whistle and clapping. ### Clean-label backdoor From 1e2904d22cc9878a6b42cdf4d705b26bccd5d018 Mon Sep 17 00:00:00 2001 From: Sterling Suggs Date: Mon, 19 Jun 2023 10:12:06 -0400 Subject: [PATCH 08/14] add link to paper for metric details --- docs/baseline_results/object_detection_poisoning_results.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/baseline_results/object_detection_poisoning_results.md b/docs/baseline_results/object_detection_poisoning_results.md index a83e68618..8aa7ae2a5 100644 --- a/docs/baseline_results/object_detection_poisoning_results.md +++ b/docs/baseline_results/object_detection_poisoning_results.md @@ -11,6 +11,8 @@ Relevant parameters: ``` (Source class is N/A for these two attacks) +See [the paper](https://arxiv.org/pdf/2205.14497.pdf) for a detailed description of each metric recorded. + # Global Misclassification Attack ## Globe trigger From d3aa975a79291f37313eb0ca7cf360edd94b085b Mon Sep 17 00:00:00 2001 From: Sterling Date: Tue, 27 Jun 2023 17:15:09 +0000 Subject: [PATCH 09/14] allow min/max size kwargs for carla multimodal frcnn --- .../pytorch/carla_multimodality_object_detection_frcnn.py | 6 ++++++ ...la_multimodality_object_detection_frcnn_robust_fusion.py | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/armory/baseline_models/pytorch/carla_multimodality_object_detection_frcnn.py b/armory/baseline_models/pytorch/carla_multimodality_object_detection_frcnn.py index dc54c3d72..4023b7c36 100755 --- a/armory/baseline_models/pytorch/carla_multimodality_object_detection_frcnn.py +++ b/armory/baseline_models/pytorch/carla_multimodality_object_detection_frcnn.py @@ -108,6 +108,11 @@ def get_art_model_mm( ) -> PyTorchFasterRCNN: num_classes = model_kwargs.pop("num_classes", 3) + frcnn_kwargs = { + arg: model_kwargs.pop(arg) + for arg in ["min_size", "max_size"] + if arg in model_kwargs + } backbone = MultimodalNaive(**model_kwargs) @@ -116,6 +121,7 @@ def get_art_model_mm( num_classes=num_classes, image_mean=[0.485, 0.456, 0.406, 0.0, 0.0, 0.0], image_std=[0.229, 0.224, 0.225, 1.0, 1.0, 1.0], + **frcnn_kwargs, ) model.to(DEVICE) diff --git a/armory/baseline_models/pytorch/carla_multimodality_object_detection_frcnn_robust_fusion.py b/armory/baseline_models/pytorch/carla_multimodality_object_detection_frcnn_robust_fusion.py index ace489f1e..7ecff0f9e 100644 --- a/armory/baseline_models/pytorch/carla_multimodality_object_detection_frcnn_robust_fusion.py +++ b/armory/baseline_models/pytorch/carla_multimodality_object_detection_frcnn_robust_fusion.py @@ -182,6 +182,11 @@ def get_art_model_mm_robust( ) -> PyTorchFasterRCNN: num_classes = model_kwargs.pop("num_classes", 3) + frcnn_kwargs = { + arg: model_kwargs.pop(arg) + for arg in ["min_size", "max_size"] + if arg in model_kwargs + } backbone = MultimodalRobust(**model_kwargs) @@ -190,6 +195,7 @@ def get_art_model_mm_robust( num_classes=num_classes, image_mean=[0.485, 0.456, 0.406, 0.0, 0.0, 0.0], image_std=[0.229, 0.224, 0.225, 1.0, 1.0, 1.0], + **frcnn_kwargs, ) model.to(DEVICE) From efbc921cf2f03d4ad382481b320ccd8da4e21f8f Mon Sep 17 00:00:00 2001 From: Sterling Date: Tue, 27 Jun 2023 19:00:12 +0000 Subject: [PATCH 10/14] update carla od configs with new kwargs --- .../carla_obj_det_adversarialpatch_undefended.json | 4 +++- .../carla_obj_det_dpatch_defended.json | 4 +++- .../carla_obj_det_dpatch_undefended.json | 4 +++- .../carla_obj_det_multimodal_adversarialpatch_defended.json | 5 ++++- ...carla_obj_det_multimodal_adversarialpatch_undefended.json | 5 ++++- .../carla_obj_det_multimodal_dpatch_defended.json | 5 ++++- .../carla_obj_det_multimodal_dpatch_undefended.json | 5 ++++- 7 files changed, 25 insertions(+), 7 deletions(-) diff --git a/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_adversarialpatch_undefended.json b/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_adversarialpatch_undefended.json index f8ff17f0f..f3733a561 100644 --- a/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_adversarialpatch_undefended.json +++ b/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_adversarialpatch_undefended.json @@ -41,7 +41,9 @@ "fit": false, "fit_kwargs": {}, "model_kwargs": { - "num_classes": 3 + "num_classes": 3, + "min_size":960, + "max_size":1280 }, "module": "armory.baseline_models.pytorch.carla_single_modality_object_detection_frcnn", "name": "get_art_model", diff --git a/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_dpatch_defended.json b/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_dpatch_defended.json index 528924b8d..a18179762 100644 --- a/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_dpatch_defended.json +++ b/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_dpatch_defended.json @@ -52,7 +52,9 @@ "fit": false, "fit_kwargs": {}, "model_kwargs": { - "num_classes": 3 + "num_classes": 3, + "min_size":960, + "max_size":1280 }, "module": "armory.baseline_models.pytorch.carla_single_modality_object_detection_frcnn", "name": "get_art_model", diff --git a/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_dpatch_undefended.json b/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_dpatch_undefended.json index 5a921899c..6863c87f3 100644 --- a/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_dpatch_undefended.json +++ b/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_dpatch_undefended.json @@ -39,7 +39,9 @@ "fit": false, "fit_kwargs": {}, "model_kwargs": { - "num_classes": 3 + "num_classes": 3, + "min_size":960, + "max_size":1280 }, "module": "armory.baseline_models.pytorch.carla_single_modality_object_detection_frcnn", "name": "get_art_model", diff --git a/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_adversarialpatch_defended.json b/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_adversarialpatch_defended.json index 0daaeb35e..e7bcc41c5 100644 --- a/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_adversarialpatch_defended.json +++ b/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_adversarialpatch_defended.json @@ -42,7 +42,10 @@ "model": { "fit": false, "fit_kwargs": {}, - "model_kwargs": {}, + "model_kwargs": { + "min_size":960, + "max_size":1280 + }, "module": "armory.baseline_models.pytorch.carla_multimodality_object_detection_frcnn_robust_fusion", "name": "get_art_model_mm_robust", "weights_file": "carla_multimodal_robust_weights_eval7and8.pt", diff --git a/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_adversarialpatch_undefended.json b/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_adversarialpatch_undefended.json index 1696d01e7..4e730c793 100644 --- a/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_adversarialpatch_undefended.json +++ b/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_adversarialpatch_undefended.json @@ -42,7 +42,10 @@ "model": { "fit": false, "fit_kwargs": {}, - "model_kwargs": {}, + "model_kwargs": { + "min_size":960, + "max_size":1280 + }, "module": "armory.baseline_models.pytorch.carla_multimodality_object_detection_frcnn", "name": "get_art_model_mm", "weights_file": "carla_multimodal_naive_weights_eval7and8.pt", diff --git a/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_dpatch_defended.json b/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_dpatch_defended.json index fa3961833..ca0ceb3ee 100644 --- a/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_dpatch_defended.json +++ b/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_dpatch_defended.json @@ -40,7 +40,10 @@ "model": { "fit": false, "fit_kwargs": {}, - "model_kwargs": {}, + "model_kwargs": { + "min_size":960, + "max_size":1280 + }, "module": "armory.baseline_models.pytorch.carla_multimodality_object_detection_frcnn_robust_fusion", "name": "get_art_model_mm_robust", "weights_file": "carla_multimodal_robust_weights_eval7and8.pt", diff --git a/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_dpatch_undefended.json b/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_dpatch_undefended.json index 0312da8f3..f7047f6be 100644 --- a/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_dpatch_undefended.json +++ b/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_dpatch_undefended.json @@ -40,7 +40,10 @@ "model": { "fit": false, "fit_kwargs": {}, - "model_kwargs": {}, + "model_kwargs": { + "min_size":960, + "max_size":1280 + }, "module": "armory.baseline_models.pytorch.carla_multimodality_object_detection_frcnn", "name": "get_art_model_mm", "weights_file": "carla_multimodal_naive_weights_eval7and8.pt", From 57d82fdcb9ea8b81d606840192546d7e0e7d2e57 Mon Sep 17 00:00:00 2001 From: Sterling Date: Wed, 28 Jun 2023 18:02:25 +0000 Subject: [PATCH 11/14] add model kwargs to carla MOT configs --- .../carla_mot/carla_mot_adversarialpatch_undefended.json | 4 +++- .../eval7/carla_mot/carla_mot_dpatch_defended.json | 4 +++- .../eval7/carla_mot/carla_mot_dpatch_undefended.json | 4 +++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/scenario_configs/eval7/carla_mot/carla_mot_adversarialpatch_undefended.json b/scenario_configs/eval7/carla_mot/carla_mot_adversarialpatch_undefended.json index 12dbc4b53..3df36d7ac 100644 --- a/scenario_configs/eval7/carla_mot/carla_mot_adversarialpatch_undefended.json +++ b/scenario_configs/eval7/carla_mot/carla_mot_adversarialpatch_undefended.json @@ -39,7 +39,9 @@ "fit": false, "fit_kwargs": {}, "model_kwargs": { - "num_classes": 2 + "num_classes": 2, + "min_size":960, + "max_size":1280 }, "module": "armory.baseline_models.pytorch.carla_mot_frcnn_byte", "name": "get_art_model", diff --git a/scenario_configs/eval7/carla_mot/carla_mot_dpatch_defended.json b/scenario_configs/eval7/carla_mot/carla_mot_dpatch_defended.json index 178c4e5ee..54807af95 100644 --- a/scenario_configs/eval7/carla_mot/carla_mot_dpatch_defended.json +++ b/scenario_configs/eval7/carla_mot/carla_mot_dpatch_defended.json @@ -49,7 +49,9 @@ "fit": false, "fit_kwargs": {}, "model_kwargs": { - "num_classes": 2 + "num_classes": 2, + "min_size":960, + "max_size":1280 }, "module": "armory.baseline_models.pytorch.carla_mot_frcnn_byte", "name": "get_art_model", diff --git a/scenario_configs/eval7/carla_mot/carla_mot_dpatch_undefended.json b/scenario_configs/eval7/carla_mot/carla_mot_dpatch_undefended.json index e89b22170..070ff58c5 100644 --- a/scenario_configs/eval7/carla_mot/carla_mot_dpatch_undefended.json +++ b/scenario_configs/eval7/carla_mot/carla_mot_dpatch_undefended.json @@ -36,7 +36,9 @@ "fit": false, "fit_kwargs": {}, "model_kwargs": { - "num_classes": 2 + "num_classes": 2, + "min_size":960, + "max_size":1280 }, "module": "armory.baseline_models.pytorch.carla_mot_frcnn_byte", "name": "get_art_model", From 20e2482d9c097dfcd2cdc8f2f027b37feff68c82 Mon Sep 17 00:00:00 2001 From: Sterling Date: Wed, 28 Jun 2023 18:11:01 +0000 Subject: [PATCH 12/14] formatting --- .../carla_mot/carla_mot_adversarialpatch_undefended.json | 6 +++--- .../eval7/carla_mot/carla_mot_dpatch_defended.json | 6 +++--- .../eval7/carla_mot/carla_mot_dpatch_undefended.json | 6 +++--- .../carla_obj_det_adversarialpatch_undefended.json | 6 +++--- .../carla_obj_det_dpatch_defended.json | 6 +++--- .../carla_obj_det_dpatch_undefended.json | 6 +++--- .../carla_obj_det_multimodal_adversarialpatch_defended.json | 4 ++-- ...arla_obj_det_multimodal_adversarialpatch_undefended.json | 4 ++-- .../carla_obj_det_multimodal_dpatch_defended.json | 4 ++-- .../carla_obj_det_multimodal_dpatch_undefended.json | 4 ++-- 10 files changed, 26 insertions(+), 26 deletions(-) diff --git a/scenario_configs/eval7/carla_mot/carla_mot_adversarialpatch_undefended.json b/scenario_configs/eval7/carla_mot/carla_mot_adversarialpatch_undefended.json index 3df36d7ac..4e6bb0993 100644 --- a/scenario_configs/eval7/carla_mot/carla_mot_adversarialpatch_undefended.json +++ b/scenario_configs/eval7/carla_mot/carla_mot_adversarialpatch_undefended.json @@ -39,9 +39,9 @@ "fit": false, "fit_kwargs": {}, "model_kwargs": { - "num_classes": 2, - "min_size":960, - "max_size":1280 + "max_size": 1280, + "min_size": 960, + "num_classes": 2 }, "module": "armory.baseline_models.pytorch.carla_mot_frcnn_byte", "name": "get_art_model", diff --git a/scenario_configs/eval7/carla_mot/carla_mot_dpatch_defended.json b/scenario_configs/eval7/carla_mot/carla_mot_dpatch_defended.json index 54807af95..aefd7ca5b 100644 --- a/scenario_configs/eval7/carla_mot/carla_mot_dpatch_defended.json +++ b/scenario_configs/eval7/carla_mot/carla_mot_dpatch_defended.json @@ -49,9 +49,9 @@ "fit": false, "fit_kwargs": {}, "model_kwargs": { - "num_classes": 2, - "min_size":960, - "max_size":1280 + "max_size": 1280, + "min_size": 960, + "num_classes": 2 }, "module": "armory.baseline_models.pytorch.carla_mot_frcnn_byte", "name": "get_art_model", diff --git a/scenario_configs/eval7/carla_mot/carla_mot_dpatch_undefended.json b/scenario_configs/eval7/carla_mot/carla_mot_dpatch_undefended.json index 070ff58c5..17f11e2b1 100644 --- a/scenario_configs/eval7/carla_mot/carla_mot_dpatch_undefended.json +++ b/scenario_configs/eval7/carla_mot/carla_mot_dpatch_undefended.json @@ -36,9 +36,9 @@ "fit": false, "fit_kwargs": {}, "model_kwargs": { - "num_classes": 2, - "min_size":960, - "max_size":1280 + "max_size": 1280, + "min_size": 960, + "num_classes": 2 }, "module": "armory.baseline_models.pytorch.carla_mot_frcnn_byte", "name": "get_art_model", diff --git a/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_adversarialpatch_undefended.json b/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_adversarialpatch_undefended.json index f3733a561..77a91860c 100644 --- a/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_adversarialpatch_undefended.json +++ b/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_adversarialpatch_undefended.json @@ -41,9 +41,9 @@ "fit": false, "fit_kwargs": {}, "model_kwargs": { - "num_classes": 3, - "min_size":960, - "max_size":1280 + "max_size": 1280, + "min_size": 960, + "num_classes": 3 }, "module": "armory.baseline_models.pytorch.carla_single_modality_object_detection_frcnn", "name": "get_art_model", diff --git a/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_dpatch_defended.json b/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_dpatch_defended.json index a18179762..b38cd6b16 100644 --- a/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_dpatch_defended.json +++ b/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_dpatch_defended.json @@ -52,9 +52,9 @@ "fit": false, "fit_kwargs": {}, "model_kwargs": { - "num_classes": 3, - "min_size":960, - "max_size":1280 + "max_size": 1280, + "min_size": 960, + "num_classes": 3 }, "module": "armory.baseline_models.pytorch.carla_single_modality_object_detection_frcnn", "name": "get_art_model", diff --git a/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_dpatch_undefended.json b/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_dpatch_undefended.json index 6863c87f3..6706e3bae 100644 --- a/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_dpatch_undefended.json +++ b/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_dpatch_undefended.json @@ -39,9 +39,9 @@ "fit": false, "fit_kwargs": {}, "model_kwargs": { - "num_classes": 3, - "min_size":960, - "max_size":1280 + "max_size": 1280, + "min_size": 960, + "num_classes": 3 }, "module": "armory.baseline_models.pytorch.carla_single_modality_object_detection_frcnn", "name": "get_art_model", diff --git a/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_adversarialpatch_defended.json b/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_adversarialpatch_defended.json index e7bcc41c5..8ad114788 100644 --- a/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_adversarialpatch_defended.json +++ b/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_adversarialpatch_defended.json @@ -43,8 +43,8 @@ "fit": false, "fit_kwargs": {}, "model_kwargs": { - "min_size":960, - "max_size":1280 + "max_size": 1280, + "min_size": 960 }, "module": "armory.baseline_models.pytorch.carla_multimodality_object_detection_frcnn_robust_fusion", "name": "get_art_model_mm_robust", diff --git a/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_adversarialpatch_undefended.json b/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_adversarialpatch_undefended.json index 4e730c793..6153fd746 100644 --- a/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_adversarialpatch_undefended.json +++ b/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_adversarialpatch_undefended.json @@ -43,8 +43,8 @@ "fit": false, "fit_kwargs": {}, "model_kwargs": { - "min_size":960, - "max_size":1280 + "max_size": 1280, + "min_size": 960 }, "module": "armory.baseline_models.pytorch.carla_multimodality_object_detection_frcnn", "name": "get_art_model_mm", diff --git a/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_dpatch_defended.json b/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_dpatch_defended.json index ca0ceb3ee..d0ba23dd6 100644 --- a/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_dpatch_defended.json +++ b/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_dpatch_defended.json @@ -41,8 +41,8 @@ "fit": false, "fit_kwargs": {}, "model_kwargs": { - "min_size":960, - "max_size":1280 + "max_size": 1280, + "min_size": 960 }, "module": "armory.baseline_models.pytorch.carla_multimodality_object_detection_frcnn_robust_fusion", "name": "get_art_model_mm_robust", diff --git a/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_dpatch_undefended.json b/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_dpatch_undefended.json index f7047f6be..c8a1c24f2 100644 --- a/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_dpatch_undefended.json +++ b/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_dpatch_undefended.json @@ -41,8 +41,8 @@ "fit": false, "fit_kwargs": {}, "model_kwargs": { - "min_size":960, - "max_size":1280 + "max_size": 1280, + "min_size": 960 }, "module": "armory.baseline_models.pytorch.carla_multimodality_object_detection_frcnn", "name": "get_art_model_mm", From 00ca9fe16599c4d40d294b1201820c1f9dcb4571 Mon Sep 17 00:00:00 2001 From: matt wartell Date: Mon, 3 Jul 2023 10:54:46 -0400 Subject: [PATCH 13/14] simplify release process release only from `master` instead of requiring a version branch like `r0.17.2` add convenience script for triggering release --- .github/workflows/release.yml | 2 -- tools/trigger-release.sh | 28 ++++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) create mode 100755 tools/trigger-release.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f975e3489..83a334995 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -35,7 +35,6 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 0 - ref: ${{ github.event.client_payload.branch }} - name: 🔨 Build wheel run: | @@ -86,7 +85,6 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 0 - ref: ${{ github.event.client_payload.branch }} - name: 🌎 Setup Build Environment run: | diff --git a/tools/trigger-release.sh b/tools/trigger-release.sh new file mode 100755 index 000000000..f0bef9cf5 --- /dev/null +++ b/tools/trigger-release.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# conveience script to trigger a build and release of Armory +# requires the environment variable ARMORY_GITHUB_TOKEN to be set + + +if [ -z "$ARMORY_GITHUB_TOKEN" ] ; then + echo "environment variable ARMORY_GITHUB_TOKEN is not set" + exit 1 +fi + + +read -e -p "Are you sure you want to trigger a Armory build and release (y/n) " choice +if [[ "$choice" != [Yy]* ]] ; then + echo "aborted." + exit 2 +fi + +echo "sending request to trigger build and release" + +curl \ + -H "Accept: application/vnd.github.everest-preview+json" \ + -H "Authorization: token $ARMORY_GITHUB_TOKEN" \ + --request POST \ + --data '{"event_type": "build-and-release"}' \ + https://api.github.com/repos/twosixlabs/armory/dispatches + +echo sent. From a9d53a62f2a9a7dade5e833a264ceeb577dff9c1 Mon Sep 17 00:00:00 2001 From: Christopher Woodall Date: Mon, 3 Jul 2023 10:07:59 -0500 Subject: [PATCH 14/14] Update trigger-release.sh --- tools/trigger-release.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/trigger-release.sh b/tools/trigger-release.sh index f0bef9cf5..3c14a1ccb 100755 --- a/tools/trigger-release.sh +++ b/tools/trigger-release.sh @@ -5,24 +5,24 @@ if [ -z "$ARMORY_GITHUB_TOKEN" ] ; then - echo "environment variable ARMORY_GITHUB_TOKEN is not set" + echo "Environment variable ARMORY_GITHUB_TOKEN is not set" exit 1 fi read -e -p "Are you sure you want to trigger a Armory build and release (y/n) " choice if [[ "$choice" != [Yy]* ]] ; then - echo "aborted." + echo "Operation Aborted." exit 2 fi -echo "sending request to trigger build and release" +echo "Sending request to trigger build and release..." curl \ - -H "Accept: application/vnd.github.everest-preview+json" \ - -H "Authorization: token $ARMORY_GITHUB_TOKEN" \ + -H "Accept: application/vnd.github.everest-preview+json" \ + -H "Authorization: token $ARMORY_GITHUB_TOKEN" \ --request POST \ --data '{"event_type": "build-and-release"}' \ https://api.github.com/repos/twosixlabs/armory/dispatches -echo sent. +echo "Request sent. Check PYPI and Docker Hub for new releases."