Merge pull request #792 from Project-MONAI/release/0.1.14

Release/0.1.14

src/Shared/Shared/ValidationConstants.cs

@@ -61,7 +61,7 @@ public static class ValidationConstants
         /// <summary>
         /// Key for the memory.
         /// </summary>
-        public static readonly string Memory = "memory_gb";
+        public static readonly string Memory = "memory";
 
         /// <summary>
         /// Key for the GPU.

src/TaskManager/Plug-ins/Argo/StaticValues/Keys.cs

@@ -88,21 +88,6 @@ internal static class Keys
         /// </summary>
         public static readonly string TaskPriorityClassName = "priority";
 
-        /// <summary>
-        /// Key for CPU
-        /// </summary>
-        public static readonly string Cpu = "cpu";
-
-        /// <summary>
-        /// Key for memory allocation
-        /// </summary>
-        public static readonly string Memory = "memory_gb";
-
-        /// <summary>
-        /// Key for GPU
-        /// </summary>
-        public static readonly string Gpu = "number_gpu";
-
         /// <summary>
         /// Required arguments to run the Argo workflow.
         /// </summary>

src/TaskManager/Plug-ins/Argo/StaticValues/ResourcesKeys.cs

@@ -14,14 +14,16 @@
  * limitations under the License.
  */
 
+using static Monai.Deploy.WorkflowManager.Shared.ValidationConstants;
+
 namespace Monai.Deploy.WorkflowManager.TaskManager.Argo.StaticValues
 {
     public static class ResourcesKeys
     {
-        public static readonly ResourcesKey GpuLimit = new() { TaskKey = "gpu_required", ArgoKey = "nvidia.com/gpu" };
+        public static readonly ResourcesKey GpuLimit = new() { TaskKey = GpuRequired, ArgoKey = "nvidia.com/gpu" };
 
-        public static readonly ResourcesKey MemoryLimit = new() { TaskKey = "memory_gb", ArgoKey = "memory" };
+        public static readonly ResourcesKey MemoryLimit = new() { TaskKey = Memory, ArgoKey = "memory" };
 
-        public static readonly ResourcesKey CpuLimit = new() { TaskKey = "cpu", ArgoKey = "cpu" };
+        public static readonly ResourcesKey CpuLimit = new() { TaskKey = Cpu, ArgoKey = "cpu" };
     }
 }

src/WorkflowManager/Common/Services/PayloadService.cs

@@ -86,7 +86,7 @@ public PayloadService(
                     CallingAeTitle = eventPayload.CallingAeTitle,
                     Timestamp = eventPayload.Timestamp,
                     PatientDetails = patientDetails,
-                    PayloadDeleted = PayloadDeleted.No,
+                    PayloadDeleted = PayloadDeleted.No
                 };
 
                 if (await _payloadRepository.CreateAsync(payload))
@@ -173,12 +173,12 @@ public async Task<bool> DeletePayloadFromStorageAsync(string payloadId)
                 throw new MonaiNotFoundException($"Payload with ID: {payloadId} not found");
             }
 
-            if (payload.PayloadDeleted == PayloadDeleted.InProgress)
+            if (payload.PayloadDeleted == PayloadDeleted.InProgress || payload.PayloadDeleted == PayloadDeleted.Yes)
             {
-                throw new MonaiBadRequestException($"Deletion of files for payload ID: {payloadId} already in progress");
+                throw new MonaiBadRequestException($"Deletion of files for payload ID: {payloadId} already in progress or already deleted");
             }
 
-            // update the payload to in progress before we request deletion form MinIO
+            // update the payload to in progress before we request deletion from storage
             payload.PayloadDeleted = PayloadDeleted.InProgress;
             await _payloadRepository.UpdateAsync(payload);
 
@@ -188,12 +188,19 @@ public async Task<bool> DeletePayloadFromStorageAsync(string payloadId)
             {
                 try
                 {
-                    await _storageService.RemoveObjectsAsync(payload.Bucket, payload.Files.Select(f => f.Path));
+                    // get all objects for the payload in storage to be deleted
+                    var allPayloadObjects = await _storageService.ListObjectsAsync(payload.Bucket, payloadId, true);
+
+                    if (allPayloadObjects.Any())
+                    {
+                        await _storageService.RemoveObjectsAsync(payload.Bucket, allPayloadObjects.Select(o => o.FilePath));
+                    }
+
                     payload.PayloadDeleted = PayloadDeleted.Yes;
                 }
-                catch
+                catch (Exception ex)
                 {
-                    _logger.PayloadUpdateFailed(payloadId);
+                    _logger.PayloadDeleteFailed(payloadId, ex);
                     payload.PayloadDeleted = PayloadDeleted.Failed;
                 }
                 finally

src/WorkflowManager/Database/Interfaces/IWorkflowRepository.cs

@@ -17,6 +17,7 @@
 using System;
 using System.Collections.Generic;
 using System.Threading.Tasks;
+using Monai.Deploy.Messaging.Events;
 using Monai.Deploy.WorkflowManager.Contracts.Models;
 
 namespace Monai.Deploy.WorkflowManager.Database.Interfaces
@@ -67,6 +68,23 @@ public interface IWorkflowRepository
         /// <param name="aeTitle">An aeTitle to retrieve workflows for.</param>
         Task<IList<WorkflowRevision>> GetWorkflowsByAeTitleAsync(List<string> aeTitles);
 
+        /// <summary>
+        /// Retrieves a list of workflows based..<br/>
+        /// if clinical workflow has AET no data origin. => WorkflowRequestEvents received with CalledAET with that AET this workflow (regardless of what the CallingAET is)<br/>
+        /// if clinical workflow has AET and data_orgins => only WorkflowRequestEvents with CalledAET with that AET  and CallingAET trigger this workflow.<br/>
+        /// </summary>
+        /// <example>
+        /// If clinical workflow (workflow revision) exists with AET “MONAI” but no data_origins set
+        /// Any inbound WorkflowRequestEvents with CalledAET = “MONAI” trigger this workflow (regardless of what the CallingAET is)
+        /// 
+        /// If clinical workflow (workflow revision) exists with AET “MONAI” and data_origins set as “PACS”
+        /// Only inbound WorkflowRequestEvents with CalledAET = “MONAI” and CallingAET = “PACS” trigger this workflow
+        /// </example>
+        /// <param name="calledAeTitle"></param>
+        /// <param name="sallingAeTitle"></param>
+        /// <returns></returns>
+        Task<IList<WorkflowRevision>> GetWorkflowsForWorkflowRequestAsync(string calledAeTitle, string callingAeTitle);
+
         /// <summary>
         /// Creates a workflow object.
         /// </summary>

src/WorkflowManager/Database/Repositories/WorkflowRepository.cs

@@ -199,9 +199,29 @@ public async Task<IList<WorkflowRevision>> GetWorkflowsByAeTitleAsync(List<strin
             return workflows;
         }
 
+        public async Task<IList<WorkflowRevision>> GetWorkflowsForWorkflowRequestAsync(string calledAeTitle, string callingAeTitle)
+        {
+            Guard.Against.NullOrEmpty(calledAeTitle);
+            Guard.Against.NullOrEmpty(callingAeTitle);
+
+            var wfs = await _workflowCollection
+                .Find(x =>
+                    x.Workflow != null &&
+                    x.Workflow.InformaticsGateway != null &&
+                    ((x.Workflow.InformaticsGateway.AeTitle == calledAeTitle &&
+                        (x.Workflow.InformaticsGateway.DataOrigins == null ||
+                        x.Workflow.InformaticsGateway.DataOrigins.Length == 0)) ||
+                    x.Workflow.InformaticsGateway.AeTitle == calledAeTitle &&
+                        x.Workflow.InformaticsGateway.DataOrigins != null &&
+                        x.Workflow.InformaticsGateway.DataOrigins.Any(d => d == callingAeTitle)) &&
+                    x.Deleted == null)
+                .ToListAsync();
+            return wfs;
+        }
+
         public async Task<string> CreateAsync(Workflow workflow)
         {
-            Guard.Against.Null(workflow, nameof(workflow));
+            Guard.Against.Null(workflow);
 
             var workflowRevision = new WorkflowRevision
             {

src/WorkflowManager/Logging/Log.300000.Payload.cs

@@ -37,5 +37,8 @@ public static partial class Log
 
         [LoggerMessage(EventId = 300005, Level = LogLevel.Error, Message = "Failed to update payload {payloadId} due to database error.")]
         public static partial void PayloadUpdateFailed(this ILogger logger, string payloadId);
+
+        [LoggerMessage(EventId = 300006, Level = LogLevel.Error, Message = "Failed to delete payload {payloadId} from storage.")]
+        public static partial void PayloadDeleteFailed(this ILogger logger, string payloadId, Exception ex);
     }
 }

src/WorkflowManager/WorkflowExecuter/Services/WorkflowExecuterService.cs

@@ -117,13 +117,8 @@ public async Task<bool> ProcessPayload(WorkflowRequestEvent message, Payload pay
             }
             else
             {
-                var aeTitles = new List<string>
-                {
-                    message.CalledAeTitle,
-                    message.CallingAeTitle
-                };
-
-                workflows = await _workflowRepository.GetWorkflowsByAeTitleAsync(aeTitles) as List<WorkflowRevision>;
+                var result = await _workflowRepository.GetWorkflowsForWorkflowRequestAsync(message.CalledAeTitle, message.CallingAeTitle);
+                workflows = new List<WorkflowRevision>(result);
             }
 
             if (workflows is null || workflows.Any() is false)

src/WorkflowManager/WorkflowManager/Validators/WorkflowValidator.cs

@@ -375,16 +375,13 @@ private void ValidateArgoTask(TaskObject currentTask)
                 }
             }
 
-            new List<string> { Cpu, Memory }.ForEach(key =>
+            if (
+                currentTask.Args.TryGetValue(Cpu, out var val) &&
+                (string.IsNullOrEmpty(val) ||
+                (double.TryParse(val, out double parsedVal) && (parsedVal < 1 || Math.Truncate(parsedVal) != parsedVal))))
             {
-                if (
-                    currentTask.Args.TryGetValue(key, out var val) &&
-                    (string.IsNullOrEmpty(val) ||
-                    (double.TryParse(val, out double parsedVal) && (parsedVal < 1 || Math.Truncate(parsedVal) != parsedVal))))
-                {
-                    Errors.Add($"Task: '{currentTask.Id}' value '{val}' provided for argument '{key}' is not valid. The value needs to be a whole number greater than 0.");
-                }
-            });
+                Errors.Add($"Task: '{currentTask.Id}' value '{val}' provided for argument '{Cpu}' is not valid. The value needs to be a whole number greater than 0.");
+            }
 
             if (
                 currentTask.Args.TryGetValue(GpuRequired, out var gpuRequired) &&

tests/IntegrationTests/WorkflowExecutor.IntegrationTests/Features/PayloadApi.feature

@@ -124,3 +124,32 @@ Scenario Outline: Get payload by Id returns 400
     When I send a GET request
     Then I will get a 400 response
     And I will receive the error message Failed to validate id, not a valid guid
+
+@DeletePayloadById
+Scenario Outline: Delete payload by Id returns 400 with invalid payload ID
+    Given I have an endpoint /payload/invalid-payload-id
+    When I send a DELETE request
+    Then I will get a 400 response
+    And I will receive the error message Failed to validate id, not a valid guid
+
+@DeletePayloadById
+Scenario Outline: Delete payload by ID returns 404 when no payload exists
+    Given I have an endpoint /payload/c5c3635b-81dd-44a9-8c3b-71adec7d47c6
+    When I send a DELETE request
+    Then I will get a 404 response
+    And I will receive the error message Payload with ID: c5c3635b-81dd-44a9-8c3b-71adec7d47c6 not found
+
+@DeletePayloadById
+Scenario Outline: Delete payload by ID returns 400 when PayloadDeleted is already InProgress
+    Given I have an endpoint /payload/c5c3635b-81dd-44a9-8c3b-71adec7d47c6
+    And I have a payload saved in mongo Payload_PayloadDeleted_InProgress
+    When I send a DELETE request
+    Then I will get a 400 response
+    And I will receive the error message Deletion of files for payload ID: c5c3635b-81dd-44a9-8c3b-71adec7d47c6 already in progress or already deleted
+
+@DeletePayloadById
+Scenario Outline: Delete payload by ID returns 202
+    Given I have an endpoint /payload/d5c3633b-41de-44a9-8c3a-71adec3d47c1
+    And I have a payload saved in mongo Payload_PayloadDeleted_No
+    When I send a DELETE request
+    Then I will get a 202 response

tests/IntegrationTests/WorkflowExecutor.IntegrationTests/Features/WorkflowRequest.feature

@@ -42,6 +42,29 @@ Scenario Outline: Publish a valid workflow request which creates multiple workfl
     | Basic_Workflow_1 | Basic_Workflow_2 | Basic_Multi_Id_WF_Request |
     | Basic_Workflow_1 | Basic_Workflow_3 | Basic_AeTitle_WF_Request  |
 
+@WorkflowRequest
+Scenario Outline: Publish a workflow request which triggers a worflow based on called_aet and calling_aet
+    Given I have a clinical workflow <workflow>
+    When I publish a Workflow Request Message <workflowRequestMessage> with no artifacts
+    Then I can see 1 Workflow Instance is created
+    And 1 Task Dispatch event is published
+    Examples:
+    | workflow                              | workflowRequestMessage            |
+    | Workflow_Called_AET                   | Called_AET_AIDE_Calling_AET_TEST  |
+    | Workflow_Called_AET_Calling_AET       | Called_AET_AIDE_Calling_AET_PACS1 |
+    | Workflow_Called_AET_Multi_Calling_AET | Called_AET_AIDE_Calling_AET_PACS1 |
+    | Workflow_Called_AET_Multi_Calling_AET | Called_AET_AIDE_Calling_AET_PACS2 |
+
+@WorkflowRequest
+Scenario Outline: Publish a workflow request which doesnt trigger a worflow based calling_aet
+    Given I have a clinical workflow <workflow>
+    When I publish a Workflow Request Message <workflowRequestMessage> with no artifacts
+    Then I can see no Workflow Instances are created
+    Examples:
+    | workflow                              | workflowRequestMessage           |
+    | Workflow_Called_AET_Calling_AET       | Called_AET_AIDE_Calling_AET_TEST |
+    | Workflow_Called_AET_Multi_Calling_AET | Called_AET_AIDE_Calling_AET_TEST |
+
 @WorkflowRequest
 Scenario: Publish a valid workflow request with mismatched AE title and workflow ID
     Given I have a clinical workflow Basic_Workflow_1

tests/IntegrationTests/WorkflowExecutor.IntegrationTests/TestData/PayloadTestData.cs

@@ -305,6 +305,54 @@ public static class PayloadsTestData
                     }
                 }
             },
+            new PayloadTestData()
+            {
+                Name = "Payload_PayloadDeleted_InProgress",
+                Payload = new Payload()
+                {
+                    Id = Guid.NewGuid().ToString(),
+                    Timestamp = DateTime.UtcNow,
+                    Bucket = "bucket_1",
+                    CalledAeTitle = "MIG",
+                    CallingAeTitle = "Basic_AE",
+                    CorrelationId = Guid.NewGuid().ToString(),
+                    PayloadId = "c5c3635b-81dd-44a9-8c3b-71adec7d47c6",
+                    Workflows = new List<string> { Guid.NewGuid().ToString() },
+                    FileCount = 50,
+                    PayloadDeleted = PayloadDeleted.InProgress,
+                    PatientDetails = new PatientDetails()
+                    {
+                        PatientDob = new DateTime(1996, 02, 05, 0, 0, 0, kind: DateTimeKind.Utc),
+                        PatientId = Guid.NewGuid().ToString(),
+                        PatientName = "Mike Mcgee",
+                        PatientSex = "male"
+                    }
+                }
+            },
+            new PayloadTestData()
+            {
+                Name = "Payload_PayloadDeleted_No",
+                Payload = new Payload()
+                {
+                    Id = Guid.NewGuid().ToString(),
+                    Timestamp = DateTime.UtcNow,
+                    Bucket = "bucket_1",
+                    CalledAeTitle = "MIG",
+                    CallingAeTitle = "Basic_AE",
+                    CorrelationId = Guid.NewGuid().ToString(),
+                    PayloadId = "d5c3633b-41de-44a9-8c3a-71adec3d47c1",
+                    Workflows = new List<string> { Guid.NewGuid().ToString() },
+                    FileCount = 50,
+                    PayloadDeleted = PayloadDeleted.No,
+                    PatientDetails = new PatientDetails()
+                    {
+                        PatientDob = new DateTime(1996, 02, 05, 0, 0, 0, kind: DateTimeKind.Utc),
+                        PatientId = Guid.NewGuid().ToString(),
+                        PatientName = "Mike Mcgee",
+                        PatientSex = "male"
+                    }
+                }
+            }
         };
     }
 }