Skip to content

[Storage][DataMovement] Fix preserving FileChangedOn during transfers #48009

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 12 commits into from
Jan 31, 2025
Merged

[Storage][DataMovement] Fix preserving FileChangedOn during transfers #48009

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
27f9222
WIP - Fix FileChangeOn during copy
jalauzon-msft Jan 29, 2025
e74dceb
Add tests, fixes
jalauzon-msft Jan 30, 2025
0fb52ae
Small fix
jalauzon-msft Jan 30, 2025
7f075fd
Fix API call, fix test, lots of recordings
jalauzon-msft Jan 30, 2025
e2e07bb
Unused imports
jalauzon-msft Jan 30, 2025
48cbbb1
Temporarily switch to project reference
jalauzon-msft Jan 30, 2025
2b87dc3
Merge branch 'main' into fix-file-changed-on
jalauzon-msft Jan 30, 2025
7445ce3
Some fixes, recordings still broken
jalauzon-msft Jan 31, 2025
747e591
Forgot some upload recordings
jalauzon-msft Jan 31, 2025
501019b
Fixed tests and re-recorded
jalauzon-msft Jan 31, 2025
5001388
Just re-record everything :)
jalauzon-msft Jan 31, 2025
51bb755
One more fix
jalauzon-msft Jan 31, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion sdk/storage/Azure.Storage.DataMovement.Blobs/assets.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,5 @@
"AssetsRepo": "Azure/azure-sdk-assets",
"AssetsRepoPrefixPath": "net",
"TagPrefix": "net/storage/Azure.Storage.DataMovement.Blobs",
"Tag": "net/storage/Azure.Storage.DataMovement.Blobs_ce7e1f61c3"
"Tag": "net/storage/Azure.Storage.DataMovement.Blobs_148b875701"
}
3 changes: 2 additions & 1 deletion sdk/storage/Azure.Storage.DataMovement.Blobs/tests/AppendBlobStartTransferCopyTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ public class AppendBlobStartTransferCopyTests : StartTransferCopyTestBase
StorageTestEnvironment>
{
private readonly AccessTier _defaultAccessTier = AccessTier.Cold;
private const string _defaultContentType = "application/octet-stream";
private const string _defaultContentType = "image/jpeg";
private const string _defaultContentLanguage = "en-US";
private const string _defaultContentDisposition = "inline";
private const string _defaultCacheControl = "no-cache";
Expand Down Expand Up @@ -233,6 +233,7 @@ protected override async Task VerifyPropertiesCopyAsync(
Assert.IsNull(destinationProperties.ContentDisposition);
Assert.IsNull(destinationProperties.ContentLanguage);
Assert.IsNull(destinationProperties.CacheControl);
Assert.That(destinationProperties.ContentType, Is.Not.EqualTo(_defaultContentType));

GetBlobTagResult destinationTags = await destinationClient.GetTagsAsync(cancellationToken: cancellationToken);
Assert.IsEmpty(destinationTags.Tags);
Expand Down
3 changes: 2 additions & 1 deletion sdk/storage/Azure.Storage.DataMovement.Blobs/tests/AppendBlobToBlockBlobTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ public class AppendBlobToBlockBlobTests : StartTransferCopyTestBase
StorageTestEnvironment>
{
private readonly AccessTier _defaultAccessTier = AccessTier.Cold;
private const string _defaultContentType = "text/plain";
private const string _defaultContentType = "image/jpeg";
private const string _defaultContentLanguage = "en-US";
private const string _defaultContentDisposition = "inline";
private const string _defaultCacheControl = "no-cache";
Expand Down Expand Up @@ -229,6 +229,7 @@ protected override async Task VerifyPropertiesCopyAsync(
Assert.IsNull(destinationProperties.ContentDisposition);
Assert.IsNull(destinationProperties.ContentLanguage);
Assert.IsNull(destinationProperties.CacheControl);
Assert.That(destinationProperties.ContentType, Is.Not.EqualTo(_defaultContentType));

GetBlobTagResult destinationTags = await destinationClient.GetTagsAsync(cancellationToken: cancellationToken);
Assert.IsEmpty(destinationTags.Tags);
Expand Down
3 changes: 2 additions & 1 deletion sdk/storage/Azure.Storage.DataMovement.Blobs/tests/AppendBlobToPageBlobTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ public class AppendBlobToPageBlobTests : StartTransferCopyTestBase
StorageTestEnvironment>
{
private readonly AccessTier _defaultAccessTier = AccessTier.Cold;
private const string _defaultContentType = "text/plain";
private const string _defaultContentType = "image/jpeg";
private const string _defaultContentLanguage = "en-US";
private const string _defaultContentDisposition = "inline";
private const string _defaultCacheControl = "no-cache";
Expand Down Expand Up @@ -248,6 +248,7 @@ protected override async Task VerifyPropertiesCopyAsync(
Assert.IsNull(destinationProperties.ContentDisposition);
Assert.IsNull(destinationProperties.ContentLanguage);
Assert.IsNull(destinationProperties.CacheControl);
Assert.That(destinationProperties.ContentType, Is.Not.EqualTo(_defaultContentType));

GetBlobTagResult destinationTags = await destinationClient.GetTagsAsync(cancellationToken: cancellationToken);
Assert.IsEmpty(destinationTags.Tags);
Expand Down
3 changes: 2 additions & 1 deletion sdk/storage/Azure.Storage.DataMovement.Blobs/tests/BlockBlobStartTransferCopyTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ public class BlockBlobStartTransferCopyTests : StartTransferCopyTestBase
StorageTestEnvironment>
{
private readonly AccessTier _defaultAccessTier = AccessTier.Cold;
private const string _defaultContentType = "text/plain";
private const string _defaultContentType = "image/jpeg";
private const string _defaultContentLanguage = "en-US";
private const string _defaultContentDisposition = "inline";
private const string _defaultCacheControl = "no-cache";
Expand Down Expand Up @@ -212,6 +212,7 @@ protected override async Task VerifyPropertiesCopyAsync(
Assert.IsNull(destinationProperties.ContentLanguage);
Assert.IsNull(destinationProperties.CacheControl);
Assert.AreEqual(_defaultAccessTier.ToString(), destinationProperties.AccessTier);
Assert.That(destinationProperties.ContentType, Is.Not.EqualTo(_defaultContentType));

GetBlobTagResult destinationTags = await destinationClient.GetTagsAsync();
Assert.IsEmpty(destinationTags.Tags);
Expand Down
3 changes: 2 additions & 1 deletion sdk/storage/Azure.Storage.DataMovement.Blobs/tests/BlockBlobToAppendBlobTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ public class BlockBlobToAppendBlobTests : StartTransferCopyTestBase
StorageTestEnvironment>
{
private readonly AccessTier _defaultAccessTier = AccessTier.Cold;
private const string _defaultContentType = "text/plain";
private const string _defaultContentType = "image/jpeg";
private const string _defaultContentLanguage = "en-US";
private const string _defaultContentDisposition = "inline";
private const string _defaultCacheControl = "no-cache";
Expand Down Expand Up @@ -233,6 +233,7 @@ protected override async Task VerifyPropertiesCopyAsync(
Assert.IsNull(destinationProperties.ContentDisposition);
Assert.IsNull(destinationProperties.ContentLanguage);
Assert.IsNull(destinationProperties.CacheControl);
Assert.That(destinationProperties.ContentType, Is.Not.EqualTo(_defaultContentType));

GetBlobTagResult destinationTags = await destinationClient.GetTagsAsync(cancellationToken: cancellationToken);
Assert.IsEmpty(destinationTags.Tags);
Expand Down
3 changes: 2 additions & 1 deletion sdk/storage/Azure.Storage.DataMovement.Blobs/tests/BlockBlobToPageBlobTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ public class BlockBlobToPageBlobTests : StartTransferCopyTestBase
StorageTestEnvironment>
{
private readonly AccessTier _defaultAccessTier = AccessTier.Cold;
private const string _defaultContentType = "text/plain";
private const string _defaultContentType = "image/jpeg";
private const string _defaultContentLanguage = "en-US";
private const string _defaultContentDisposition = "inline";
private const string _defaultCacheControl = "no-cache";
Expand Down Expand Up @@ -234,6 +234,7 @@ protected override async Task VerifyPropertiesCopyAsync(
Assert.IsNull(destinationProperties.ContentDisposition);
Assert.IsNull(destinationProperties.ContentLanguage);
Assert.IsNull(destinationProperties.CacheControl);
Assert.That(destinationProperties.ContentType, Is.Not.EqualTo(_defaultContentType));

GetBlobTagResult destinationTags = await destinationClient.GetTagsAsync();
Assert.IsEmpty(destinationTags.Tags);
Expand Down
3 changes: 2 additions & 1 deletion sdk/storage/Azure.Storage.DataMovement.Blobs/tests/PageBlobStartTransferCopyTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ public class PageBlobStartTransferCopyTests : StartTransferCopyTestBase
StorageTestEnvironment>
{
private readonly AccessTier _defaultAccessTier = AccessTier.Cool;
private const string _defaultContentType = "application/octet-stream";
private const string _defaultContentType = "image/jpeg";
private const string _defaultContentLanguage = "en-US";
private const string _defaultContentDisposition = "inline";
private const string _defaultCacheControl = "no-cache";
Expand Down Expand Up @@ -224,6 +224,7 @@ protected override async Task VerifyPropertiesCopyAsync(
Assert.IsNull(destinationProperties.ContentDisposition);
Assert.IsNull(destinationProperties.ContentLanguage);
Assert.IsNull(destinationProperties.CacheControl);
Assert.That(destinationProperties.ContentType, Is.Not.EqualTo(_defaultContentType));

GetBlobTagResult destinationTags = await destinationClient.GetTagsAsync();
Assert.IsEmpty(destinationTags.Tags);
Expand Down
3 changes: 2 additions & 1 deletion sdk/storage/Azure.Storage.DataMovement.Blobs/tests/PageBlobToAppendBlobTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ public class PageBlobToAppendBlobTests : StartTransferCopyTestBase
StorageTestEnvironment>
{
private readonly AccessTier _defaultAccessTier = AccessTier.Cold;
private const string _defaultContentType = "text/plain";
private const string _defaultContentType = "image/jpeg";
private const string _defaultContentLanguage = "en-US";
private const string _defaultContentDisposition = "inline";
private const string _defaultCacheControl = "no-cache";
Expand Down Expand Up @@ -241,6 +241,7 @@ protected override async Task VerifyPropertiesCopyAsync(
Assert.IsNull(destinationProperties.ContentDisposition);
Assert.IsNull(destinationProperties.ContentLanguage);
Assert.IsNull(destinationProperties.CacheControl);
Assert.That(destinationProperties.ContentType, Is.Not.EqualTo(_defaultContentType));

GetBlobTagResult destinationTags = await destinationClient.GetTagsAsync(cancellationToken: cancellationToken);
Assert.IsEmpty(destinationTags.Tags);
Expand Down
3 changes: 2 additions & 1 deletion sdk/storage/Azure.Storage.DataMovement.Blobs/tests/PageBlobToBlockBlobTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ public class PageBlobToBlockBlobTests : StartTransferCopyTestBase
StorageTestEnvironment>
{
private readonly AccessTier _defaultAccessTier = AccessTier.Cold;
private const string _defaultContentType = "text/plain";
private const string _defaultContentType = "image/jpeg";
private const string _defaultContentLanguage = "en-US";
private const string _defaultContentDisposition = "inline";
private const string _defaultCacheControl = "no-cache";
Expand Down Expand Up @@ -232,6 +232,7 @@ protected override async Task VerifyPropertiesCopyAsync(
Assert.IsNull(destinationProperties.ContentDisposition);
Assert.IsNull(destinationProperties.ContentLanguage);
Assert.IsNull(destinationProperties.CacheControl);
Assert.That(destinationProperties.ContentType, Is.Not.EqualTo(_defaultContentType));

GetBlobTagResult destinationTags = await destinationClient.GetTagsAsync();
Assert.IsEmpty(destinationTags.Tags);
Expand Down
2 changes: 1 addition & 1 deletion sdk/storage/Azure.Storage.DataMovement.Files.Shares/assets.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,5 @@
"AssetsRepo": "Azure/azure-sdk-assets",
"AssetsRepoPrefixPath": "net",
"TagPrefix": "net/storage/Azure.Storage.DataMovement.Files.Shares",
"Tag": "net/storage/Azure.Storage.DataMovement.Files.Shares_91bd99f1ca"
"Tag": "net/storage/Azure.Storage.DataMovement.Files.Shares_1510e7b5ae"
}
29 changes: 29 additions & 0 deletions sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/DataMovementSharesExtensions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,35 @@ public static FileSmbProperties GetFileSmbProperties(
};
}

public static FileSmbProperties GetFileSmbProperties(
this ShareFileStorageResourceOptions options,
StorageResourceItemProperties properties)
{
return new()
{
FileAttributes = (options?._isFileAttributesSet ?? false)
? options?.FileAttributes
: properties?.RawProperties?.TryGetValue(DataMovementConstants.ResourceProperties.FileAttributes, out object fileAttributes) == true
? (NtfsFileAttributes?)fileAttributes
: default,
FileCreatedOn = (options?._isFileCreatedOnSet ?? false)
? options?.FileCreatedOn
: properties?.RawProperties?.TryGetValue(DataMovementConstants.ResourceProperties.CreationTime, out object fileCreatedOn) == true
? (DateTimeOffset?)fileCreatedOn
: default,
FileLastWrittenOn = (options?._isFileLastWrittenOnSet ?? false)
? options?.FileLastWrittenOn
: properties?.RawProperties?.TryGetValue(DataMovementConstants.ResourceProperties.LastWrittenOn, out object fileLastWrittenOn) == true
? (DateTimeOffset?)fileLastWrittenOn
: default,
FileChangedOn = (options?._isFileChangedOnSet ?? false)
? options?.FileChangedOn
: properties?.RawProperties?.TryGetValue(DataMovementConstants.ResourceProperties.ChangedOnTime, out object fileChangedOn) == true
? (DateTimeOffset?)fileChangedOn
: default,
};
}

internal static ShareFileUploadRangeOptions ToShareFileUploadRangeOptions(
this ShareFileStorageResourceOptions options)
=> new()
Expand Down
18 changes: 16 additions & 2 deletions sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileStorageResource.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,13 +90,27 @@ await ShareFileClient.CreateAsync(
cancellationToken: cancellationToken).ConfigureAwait(false);
}

protected override Task CompleteTransferAsync(
protected override async Task CompleteTransferAsync(
bool overwrite,
StorageResourceCompleteTransferOptions completeTransferOptions,
CancellationToken cancellationToken = default)
{
CancellationHelper.ThrowIfCancellationRequested(cancellationToken);
return Task.CompletedTask;

// Call Set Properties if FileChangedOn is to be preserved or manually set
// as it can be changed during a transfer.
if (_options?._isFileChangedOnSet == false || _options?.FileChangedOn != null)
{
StorageResourceItemProperties sourceProperties = completeTransferOptions?.SourceProperties;
ShareFileHttpHeaders httpHeaders = _options?.GetShareFileHttpHeaders(sourceProperties?.RawProperties);
FileSmbProperties smbProperties = _options?.GetFileSmbProperties(sourceProperties);
await ShareFileClient.SetHttpHeadersAsync(new()
{
HttpHeaders = httpHeaders,
SmbProperties = smbProperties,
},
cancellationToken: cancellationToken).ConfigureAwait(false);
}
}

protected override async Task CopyBlockFromUriAsync(
Expand Down
69 changes: 3 additions & 66 deletions ...age/Azure.Storage.DataMovement.Files.Shares/tests/ShareDirectoryStartTransferCopyTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using Azure.Core.TestFramework;
using Azure.Storage.DataMovement.Tests;
using BaseShares::Azure.Storage.Files.Shares;
using BaseShares::Azure.Storage.Files.Shares.Models;
Expand Down Expand Up @@ -285,6 +284,7 @@ protected async Task VerifyPropertiesCopyAsync(
Assert.AreEqual(_defaultContentType, destinationProperties.ContentType);
Assert.AreEqual(_defaultFileCreatedOn, destinationProperties.SmbProperties.FileCreatedOn);
Assert.AreEqual(_defaultFileLastWrittenOn, destinationProperties.SmbProperties.FileLastWrittenOn);
Assert.AreEqual(_defaultFileChangedOn, destinationProperties.SmbProperties.FileChangedOn);
}
else if (transferPropertiesTestType == TransferPropertiesTestType.Preserve)
{
Expand All @@ -298,6 +298,7 @@ protected async Task VerifyPropertiesCopyAsync(
Assert.AreEqual(sourceProperties.ContentType, destinationProperties.ContentType);
Assert.AreEqual(sourceProperties.SmbProperties.FileCreatedOn, destinationProperties.SmbProperties.FileCreatedOn);
Assert.AreEqual(sourceProperties.SmbProperties.FileLastWrittenOn, destinationProperties.SmbProperties.FileLastWrittenOn);
Assert.AreEqual(sourceProperties.SmbProperties.FileChangedOn, destinationProperties.SmbProperties.FileChangedOn);

// Check if the permissions are the same. Permission Keys will be different as they are defined by the share service.
ShareClient sourceShareClient = sourceClient.GetParentShareClient();
Expand All @@ -319,6 +320,7 @@ protected async Task VerifyPropertiesCopyAsync(
Assert.AreEqual(sourceProperties.ContentType, destinationProperties.ContentType);
Assert.AreEqual(sourceProperties.SmbProperties.FileCreatedOn, destinationProperties.SmbProperties.FileCreatedOn);
Assert.AreEqual(sourceProperties.SmbProperties.FileLastWrittenOn, destinationProperties.SmbProperties.FileLastWrittenOn);
Assert.AreEqual(sourceProperties.SmbProperties.FileChangedOn, destinationProperties.SmbProperties.FileChangedOn);
}
}

Expand Down Expand Up @@ -364,70 +366,5 @@ private ShareFileStorageResourceOptions GetShareFileStorageResourceOptions(Trans
}
return options;
}

private async Task CopyRemoteObjects_VerifyProperties(
ShareClient sourceContainer,
ShareClient destinationContainer,
TransferPropertiesTestType propertiesType)
{
// Arrange
int size = Constants.KB;
string sourcePrefix = "sourceFolder";
string destPrefix = "destFolder";
await CreateDirectoryInSourceAsync(sourceContainer, sourcePrefix);
string itemName1 = string.Join("/", sourcePrefix, GetNewObjectName());
string itemName2 = string.Join("/", sourcePrefix, GetNewObjectName());
await CreateShareFileAsync(sourceContainer, size, itemName1, propertiesType: propertiesType);
await CreateShareFileAsync(sourceContainer, size, itemName2, propertiesType: propertiesType);

string subDirName = string.Join("/", sourcePrefix, "bar");
await CreateDirectoryInSourceAsync(sourceContainer, subDirName);
string itemName3 = string.Join("/", subDirName, GetNewObjectName());
await CreateShareFileAsync(sourceContainer, size, itemName3, propertiesType: propertiesType);

string subDirName2 = string.Join("/", sourcePrefix, "pik");
await CreateDirectoryInSourceAsync(sourceContainer, subDirName2);
string itemName4 = string.Join("/", subDirName2, GetNewObjectName());
await CreateShareFileAsync(sourceContainer, size, itemName4, propertiesType: propertiesType);

await CreateDirectoryInDestinationAsync(destinationContainer, destPrefix);

// Create storage resource containers
StorageResourceContainer sourceResource =
GetSourceStorageResourceContainer(sourceContainer, sourcePrefix);
StorageResourceContainer destinationResource =
GetDestinationStorageResourceContainer(destinationContainer, destPrefix, propertiesType);

// Create Transfer Manager
TransferOptions options = new TransferOptions();
TestEventsRaised testEventsRaised = new TestEventsRaised(options);
TransferManager transferManager = new TransferManager();

// Start transfer and await for completion.
TransferOperation transfer = await transferManager.StartTransferAsync(
sourceResource,
destinationResource,
options).ConfigureAwait(false);

CancellationTokenSource tokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30));
await TestTransferWithTimeout.WaitForCompletionAsync(
transfer,
testEventsRaised,
tokenSource.Token);

// Verify completion
Assert.NotNull(transfer);
Assert.IsTrue(transfer.HasCompleted);
Assert.AreEqual(TransferState.Completed, transfer.Status.State);
await testEventsRaised.AssertContainerCompletedCheck(4);

// Assert
await VerifyResultsAsync(
sourceContainer,
sourcePrefix,
destinationContainer,
destPrefix,
propertiesType);
}
}
}
Loading