Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Enable External Store use with Update Operation #2914

Merged
merged 81 commits into from
Aug 12, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
81 commits
Select commit Hold shift + click to select a range
28542cf
rm separate partitions fields from InstanceIdentifier and enforce req…
esmadau Jun 16, 2023
df66751
rm created date
esmadau Jun 16, 2023
0656d3d
rename PartitionEntry to Partition
esmadau Jun 16, 2023
36c5fbf
use default when none passed in
esmadau Jun 16, 2023
9141865
Rename to Partition, use key and name without partition prefix
esmadau Jun 19, 2023
2eba221
fix tests
esmadau Jun 20, 2023
e5aa4dd
rm sets
esmadau Jun 20, 2023
3f90f1e
manipulate swagger and response contract on partition to maintain pre…
esmadau Jun 22, 2023
1e2d431
fix nits
esmadau Jun 22, 2023
63258f2
use partition off of instance
esmadau Jun 22, 2023
6082276
Update src/Microsoft.Health.Dicom.Client/Models/Partition.cs
esmadau Jun 22, 2023
b3b1682
rm no longer needed DefaultPartition, use DefaultName and DefaultKey
esmadau Jun 22, 2023
19dce06
use singleton
esmadau Jun 22, 2023
ebaddc8
Merge branch 'main' into users/esmadau/partcln1
esmadau Jul 6, 2023
3ff412e
keep same serialization casing
esmadau Jul 6, 2023
83ae931
comments
esmadau Jul 6, 2023
0ea7dc9
Merge branch 'main' into users/esmadau/partcln1
esmadau Jul 7, 2023
9f4bdaa
fix tests
esmadau Jul 7, 2023
504cf5f
Merge branch 'main' into users/esmadau/partcln1
esmadau Jul 7, 2023
c1ed723
fix merge dangler
esmadau Jul 7, 2023
73319f3
fix nits
esmadau Jul 10, 2023
73362f5
delete no longer needed ctor on VersionedInstanceIdentifier
esmadau Jul 10, 2023
c7f5927
fix nit
esmadau Jul 10, 2023
d503439
do not break functions
esmadau Jul 10, 2023
d25d6da
rm extra mock
esmadau Jul 10, 2023
ef6e696
rm unneeded file
esmadau Jul 11, 2023
f121864
Merge branch 'main' into users/esmadau/partcln1
esmadau Jul 11, 2023
1989440
continue to use CreatedDate
esmadau Jul 12, 2023
d88c333
enable update operation to interact with external store
esmadau Jul 13, 2023
951be9c
use updated instances to get new watermark for file props in test
esmadau Jul 14, 2023
3deeef0
bump to schema version 44 in prep to rebase with main
esmadau Jul 25, 2023
8077e4c
Merge branch 'main' into users/esmadau/idp-update
esmadau Jul 25, 2023
bcf4653
insert file properties and add path to change feed on Update operatio…
esmadau Jul 25, 2023
9e4bc6c
stop using unknownname on partition
esmadau Jul 26, 2023
5d42cbd
rm UpdateInstanceWatermarkV2Async and use of unknown name for partition
esmadau Jul 26, 2023
6c45149
rm UpdateInstanceBlobsV2Async
esmadau Jul 26, 2023
c0faab5
rm CompleteUpdateStudyV2Async
esmadau Jul 26, 2023
1e0169d
rm DeleteOldVersionBlobV2Async
esmadau Jul 26, 2023
389be55
rm CleanupNewVersionBlobV2Async
esmadau Jul 26, 2023
9422192
rm CompleteUpdateStudyV2Async
esmadau Jul 26, 2023
586c3b9
don't rely on change feed order of insertion when multiple tests insert
esmadau Jul 27, 2023
bff7955
rm unknown name from UpdateInstanceBlobArguments
esmadau Jul 27, 2023
b3dce19
added test
esmadau Aug 1, 2023
030fc28
rm redundant code
esmadau Aug 1, 2023
ad46fcc
cleanup
esmadau Aug 1, 2023
2743df9
cleanup
esmadau Aug 1, 2023
fbdce52
added tests and helping comment to make it more visible that delete c…
esmadau Aug 1, 2023
32e8181
added newlines
esmadau Aug 1, 2023
e570f02
enable external store on update
esmadau Aug 1, 2023
6e07161
rm comment
esmadau Aug 1, 2023
a3e0180
move external store check up to orchestrator
esmadau Aug 2, 2023
d0a48cf
fix nits from review
esmadau Aug 2, 2023
d6de523
delete stale file properties when doing update operation
esmadau Aug 4, 2023
2bc64de
move changefeed related tests to changefeed test class
esmadau Aug 7, 2023
a39b4a3
start using and registering external store feature settings for funct…
esmadau Aug 7, 2023
1ee0dd8
use feature config within update function
esmadau Aug 7, 2023
1364b57
version update operation functions
esmadau Aug 7, 2023
4de9f76
add indices on temp and file property type tables on watermark
esmadau Aug 7, 2023
1f465ec
do only single index, after loading data
esmadau Aug 8, 2023
657f825
Use InstanceMetadata to insert file property rows instead of custom obj
esmadau Aug 9, 2023
fa5cca9
fix nits
esmadau Aug 9, 2023
9bf6160
split orchestration between v1 and v2 as returned info from v2 functi…
esmadau Aug 9, 2023
682b260
fix nits
esmadau Aug 9, 2023
d735bdf
mark all v1 related fields and methods as obsolete to remove in follo…
esmadau Aug 9, 2023
6dd4d10
fix nits
esmadau Aug 9, 2023
fc14910
fix nits
esmadau Aug 9, 2023
5779377
fix nits
esmadau Aug 9, 2023
2069261
use newer sql tools so temp table can be parsed
esmadau Aug 10, 2023
3cdb66c
fix nits
esmadau Aug 10, 2023
001452a
fix nits
esmadau Aug 10, 2023
71ba162
Merge branch 'main' into users/esmadau/idp-update
esmadau Aug 10, 2023
35f3148
fix nits
esmadau Aug 10, 2023
323866b
finish mv metadatas <> metadataList
esmadau Aug 10, 2023
ed285ab
use v2 in test
esmadau Aug 10, 2023
a24d861
create another index on UpdatedInstance to include key and original w…
esmadau Aug 10, 2023
184b9a1
update Update fn names in config
esmadau Aug 10, 2023
9f7d39a
Merge branch 'main' into users/esmadau/idp-update
esmadau Aug 10, 2023
074716f
update statement uses watermarks after insertion instead of before to…
esmadau Aug 10, 2023
9d06a7f
added index comments
esmadau Aug 11, 2023
ff34141
select index FROM tempdb.sys.indexes
esmadau Aug 11, 2023
4e6fa59
fix nit
esmadau Aug 11, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,6 @@ public class StoreController : ControllerBase
private readonly ILogger<StoreController> _logger;
private readonly bool _dicomUpdateEnabled;
private readonly bool _dataPartitionsEnabled;
private readonly bool _externalStoreEnabled;

public StoreController(IMediator mediator, ILogger<StoreController> logger, IOptions<FeatureConfiguration> featureConfiguration)
{
Expand All @@ -51,7 +50,6 @@ public StoreController(IMediator mediator, ILogger<StoreController> logger, IOpt
_logger = logger;
_dicomUpdateEnabled = featureConfiguration.Value.EnableUpdate;
_dataPartitionsEnabled = featureConfiguration.Value.EnableDataPartitions;
_externalStoreEnabled = featureConfiguration.Value.EnableExternalStore;
}

[AcceptContentFilter(new[] { KnownContentTypes.ApplicationDicomJson })]
Expand Down Expand Up @@ -108,11 +106,6 @@ public async Task<IActionResult> UpdateAsync([FromBody][Required] UpdateSpecific
throw new DicomUpdateFeatureDisabledException();
}

if (_externalStoreEnabled)
{
throw new DicomUpdateFeatureDisabledException();
}

UpdateInstanceResponse response = await _mediator.UpdateInstanceAsync(updateSpecification);
if (response.FailedDataset != null)
{
Expand Down
42 changes: 26 additions & 16 deletions src/Microsoft.Health.Dicom.Blob/Features/Storage/BlobFileStore.cs
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
using Microsoft.Health.Dicom.Core.Exceptions;
using Microsoft.Health.Dicom.Core.Features.Common;
using Microsoft.Health.Dicom.Core.Features.Model;
using Microsoft.Health.Dicom.Core.Features.Partitioning;

namespace Microsoft.Health.Dicom.Blob.Features.Storage;

Expand Down Expand Up @@ -72,16 +73,17 @@ public async Task<FileProperties> StoreFileAsync(
/// <inheritdoc />
public async Task<Uri> StoreFileInBlocksAsync(
long version,
Partition partition,
Stream stream,
IDictionary<string, long> blockLengths,
CancellationToken cancellationToken)
{
EnsureArg.IsNotNull(stream, nameof(stream));
EnsureArg.IsNotNull(partition, nameof(partition));
EnsureArg.IsNotNull(blockLengths, nameof(blockLengths));
EnsureArg.IsGte(blockLengths.Count, 0, nameof(blockLengths.Count));

// pass in partition name once update supported by external store #104373
BlockBlobClient blobClient = GetInstanceBlockBlobClient(version, string.Empty);
BlockBlobClient blobClient = GetInstanceBlockBlobClient(version, partition.Name);

int maxBufferSize = (int)blockLengths.Max(x => x.Value);

Expand Down Expand Up @@ -111,17 +113,18 @@ public async Task<Uri> StoreFileInBlocksAsync(
}

/// <inheritdoc />
public async Task UpdateFileBlockAsync(
public async Task<FileProperties> UpdateFileBlockAsync(
long version,
Partition partition,
string blockId,
Stream stream,
CancellationToken cancellationToken)
{
EnsureArg.IsNotNull(stream, nameof(stream));
EnsureArg.IsNotNull(partition, nameof(partition));
EnsureArg.IsNotNullOrWhiteSpace(blockId, nameof(blockId));

// pass in partition name once update supported by external store #104373
BlockBlobClient blobClient = GetInstanceBlockBlobClient(version, string.Empty);
BlockBlobClient blobClient = GetInstanceBlockBlobClient(version, partition.Name);
_logger.LogInformation("Trying to read block list for DICOM instance file with watermark '{Version}'.", version);

BlockList blockList = await ExecuteAsync<BlockList>(async () => await blobClient.GetBlockListAsync(
Expand All @@ -139,11 +142,18 @@ public async Task UpdateFileBlockAsync(

stream.Seek(0, SeekOrigin.Begin);

await ExecuteAsync(async () =>
BlobContentInfo info = await ExecuteAsync(async () =>
{
await blobClient.StageBlockAsync(blockId, stream, cancellationToken: cancellationToken);
return await blobClient.CommitBlockListAsync(blockIds, cancellationToken: cancellationToken);
});

return new FileProperties
{
Path = blobClient.Name,
ETag = info.ETag.ToString(),
ContentLength = stream.Length
};
}

/// <inheritdoc />
Expand Down Expand Up @@ -220,11 +230,11 @@ public async Task<Stream> GetFileFrameAsync(long version, string partitionName,
}

/// <inheritdoc />
public async Task<BinaryData> GetFileContentInRangeAsync(long version, FrameRange range, CancellationToken cancellationToken)
public async Task<BinaryData> GetFileContentInRangeAsync(long version, Partition partition, FrameRange range, CancellationToken cancellationToken)
{
EnsureArg.IsNotNull(range, nameof(range));
// pass in partition name once update supported by external store #104373
BlockBlobClient blob = GetInstanceBlockBlobClient(version, string.Empty);
EnsureArg.IsNotNull(partition, nameof(partition));
BlockBlobClient blob = GetInstanceBlockBlobClient(version, partition.Name);
_logger.LogInformation("Trying to read DICOM instance fileContent with watermark '{Version}' on range {Offset}-{Length}.", version, range.Offset, range.Length);

var blobDownloadOptions = new BlobDownloadOptions
Expand All @@ -240,10 +250,10 @@ public async Task<BinaryData> GetFileContentInRangeAsync(long version, FrameRang
}

/// <inheritdoc />
public async Task<KeyValuePair<string, long>> GetFirstBlockPropertyAsync(long version, CancellationToken cancellationToken = default)
public async Task<KeyValuePair<string, long>> GetFirstBlockPropertyAsync(long version, Partition partition, CancellationToken cancellationToken = default)
{
// pass in partition name once update supported by external store #104373
BlockBlobClient blobClient = GetInstanceBlockBlobClient(version, string.Empty);
EnsureArg.IsNotNull(partition, nameof(partition));
BlockBlobClient blobClient = GetInstanceBlockBlobClient(version, partition.Name);
_logger.LogInformation("Trying to read DICOM instance file with watermark '{Version}' firstBlock.", version);

return await ExecuteAsync(async () =>
Expand All @@ -259,11 +269,11 @@ public async Task<KeyValuePair<string, long>> GetFirstBlockPropertyAsync(long ve
}

/// <inheritdoc />
public async Task CopyFileAsync(long originalVersion, long newVersion, CancellationToken cancellationToken)
public async Task CopyFileAsync(long originalVersion, long newVersion, Partition partition, CancellationToken cancellationToken)
{
// pass in partition name once update supported by external store #104373
var blobClient = GetInstanceBlockBlobClient(originalVersion, string.Empty);
var copyBlobClient = GetInstanceBlockBlobClient(newVersion, string.Empty);
EnsureArg.IsNotNull(partition, nameof(partition));
var blobClient = GetInstanceBlockBlobClient(originalVersion, partition.Name);
var copyBlobClient = GetInstanceBlockBlobClient(newVersion, partition.Name);
_logger.LogInformation("Trying to copy DICOM instance file with watermark '{Version}' to new watermark '{NewVersion}.", originalVersion, newVersion);

if (!await copyBlobClient.ExistsAsync(cancellationToken))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,10 +39,10 @@ public class StoreOrchestratorTests
DefaultSopInstanceUid,
DefaultVersion);

private static readonly FileProperties DefaultFileProperties = new FileProperties()
private static readonly FileProperties DefaultFileProperties = new FileProperties
{
Path = String.Empty,
ETag = String.Empty
Path = "default/path/0.dcm",
ETag = "123"
};

private static readonly CancellationToken DefaultCancellationToken = new CancellationTokenSource().Token;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ public async Task GivenValidInput_WhenNoExistingOperationQueued_ThenShouldSuccee
.StartUpdateOperationAsync(
Arg.Any<Guid>(),
Arg.Any<UpdateSpecification>(),
Partition.DefaultKey,
Partition.Default,
CancellationToken.None)
.Returns(expected);

Expand Down
Loading
Loading