[Storage][DataMovement] Fix preserving FileChangedOn during transfers

sdk/storage/Azure.Storage.DataMovement.Blobs/assets.json

@@ -2,5 +2,5 @@
   "AssetsRepo": "Azure/azure-sdk-assets",
   "AssetsRepoPrefixPath": "net",
   "TagPrefix": "net/storage/Azure.Storage.DataMovement.Blobs",
-  "Tag": "net/storage/Azure.Storage.DataMovement.Blobs_ce7e1f61c3"
+  "Tag": "net/storage/Azure.Storage.DataMovement.Blobs_f4aa54a095"
 }

sdk/storage/Azure.Storage.DataMovement.Files.Shares/assets.json

@@ -2,5 +2,5 @@
   "AssetsRepo": "Azure/azure-sdk-assets",
   "AssetsRepoPrefixPath": "net",
   "TagPrefix": "net/storage/Azure.Storage.DataMovement.Files.Shares",
-  "Tag": "net/storage/Azure.Storage.DataMovement.Files.Shares_91bd99f1ca"
+  "Tag": "net/storage/Azure.Storage.DataMovement.Files.Shares_1510e7b5ae"
 }

sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/DataMovementSharesExtensions.cs

@@ -118,6 +118,35 @@ public static FileSmbProperties GetFileSmbProperties(
             };
         }
 
+        public static FileSmbProperties GetFileSmbProperties(
+            this ShareFileStorageResourceOptions options,
+            StorageResourceItemProperties properties)
+        {
+            return new()
+            {
+                FileAttributes = (options?._isFileAttributesSet ?? false)
+                    ? options?.FileAttributes
+                    : properties?.RawProperties?.TryGetValue(DataMovementConstants.ResourceProperties.FileAttributes, out object fileAttributes) == true
+                        ? (NtfsFileAttributes?)fileAttributes
+                        : default,
+                FileCreatedOn = (options?._isFileCreatedOnSet ?? false)
+                    ? options?.FileCreatedOn
+                    : properties?.RawProperties?.TryGetValue(DataMovementConstants.ResourceProperties.CreationTime, out object fileCreatedOn) == true
+                        ? (DateTimeOffset?)fileCreatedOn
+                        : default,
+                FileLastWrittenOn = (options?._isFileLastWrittenOnSet ?? false)
+                    ? options?.FileLastWrittenOn
+                    : properties?.RawProperties?.TryGetValue(DataMovementConstants.ResourceProperties.LastWrittenOn, out object fileLastWrittenOn) == true
+                        ? (DateTimeOffset?)fileLastWrittenOn
+                        : default,
+                FileChangedOn = (options?._isFileChangedOnSet ?? false)
+                    ? options?.FileChangedOn
+                    : properties?.RawProperties?.TryGetValue(DataMovementConstants.ResourceProperties.ChangedOnTime, out object fileChangedOn) == true
+                        ? (DateTimeOffset?)fileChangedOn
+                        : default,
+            };
+        }
+
         internal static ShareFileUploadRangeOptions ToShareFileUploadRangeOptions(
             this ShareFileStorageResourceOptions options)
             => new()

sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileStorageResource.cs

@@ -90,13 +90,27 @@ await ShareFileClient.CreateAsync(
                     cancellationToken: cancellationToken).ConfigureAwait(false);
         }
 
-        protected override Task CompleteTransferAsync(
+        protected override async Task CompleteTransferAsync(
             bool overwrite,
             StorageResourceCompleteTransferOptions completeTransferOptions,
             CancellationToken cancellationToken = default)
         {
             CancellationHelper.ThrowIfCancellationRequested(cancellationToken);
-            return Task.CompletedTask;
+
+            // Call Set Properties if FileChangedOn is to be preserved or manually set
+            // as it can be changed during a transfer.
+            if (_options?._isFileChangedOnSet == false || _options?.FileChangedOn != null)
+            {
+                StorageResourceItemProperties sourceProperties = completeTransferOptions?.SourceProperties;
+                ShareFileHttpHeaders httpHeaders = _options?.GetShareFileHttpHeaders(sourceProperties?.RawProperties);
+                FileSmbProperties smbProperties = _options?.GetFileSmbProperties(sourceProperties);
+                await ShareFileClient.SetHttpHeadersAsync(new()
+                {
+                    HttpHeaders = httpHeaders,
+                    SmbProperties = smbProperties,
+                },
+                cancellationToken: cancellationToken).ConfigureAwait(false);
+            }
         }
 
         protected override async Task CopyBlockFromUriAsync(

sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareDirectoryStartTransferCopyTests.cs

@@ -7,7 +7,6 @@
 using System.IO;
 using System.Linq;
 using System.Threading.Tasks;
-using Azure.Core.TestFramework;
 using Azure.Storage.DataMovement.Tests;
 using BaseShares::Azure.Storage.Files.Shares;
 using BaseShares::Azure.Storage.Files.Shares.Models;
@@ -285,6 +284,7 @@ protected async Task VerifyPropertiesCopyAsync(
                 Assert.AreEqual(_defaultContentType, destinationProperties.ContentType);
                 Assert.AreEqual(_defaultFileCreatedOn, destinationProperties.SmbProperties.FileCreatedOn);
                 Assert.AreEqual(_defaultFileLastWrittenOn, destinationProperties.SmbProperties.FileLastWrittenOn);
+                Assert.AreEqual(_defaultFileChangedOn, destinationProperties.SmbProperties.FileChangedOn);
             }
             else if (transferPropertiesTestType == TransferPropertiesTestType.Preserve)
             {
@@ -298,6 +298,7 @@ protected async Task VerifyPropertiesCopyAsync(
                 Assert.AreEqual(sourceProperties.ContentType, destinationProperties.ContentType);
                 Assert.AreEqual(sourceProperties.SmbProperties.FileCreatedOn, destinationProperties.SmbProperties.FileCreatedOn);
                 Assert.AreEqual(sourceProperties.SmbProperties.FileLastWrittenOn, destinationProperties.SmbProperties.FileLastWrittenOn);
+                Assert.AreEqual(sourceProperties.SmbProperties.FileChangedOn, destinationProperties.SmbProperties.FileChangedOn);
 
                 // Check if the permissions are the same. Permission Keys will be different as they are defined by the share service.
                 ShareClient sourceShareClient = sourceClient.GetParentShareClient();
@@ -319,6 +320,7 @@ protected async Task VerifyPropertiesCopyAsync(
                 Assert.AreEqual(sourceProperties.ContentType, destinationProperties.ContentType);
                 Assert.AreEqual(sourceProperties.SmbProperties.FileCreatedOn, destinationProperties.SmbProperties.FileCreatedOn);
                 Assert.AreEqual(sourceProperties.SmbProperties.FileLastWrittenOn, destinationProperties.SmbProperties.FileLastWrittenOn);
+                Assert.AreEqual(sourceProperties.SmbProperties.FileChangedOn, destinationProperties.SmbProperties.FileChangedOn);
             }
         }
 
@@ -364,70 +366,5 @@ private ShareFileStorageResourceOptions GetShareFileStorageResourceOptions(Trans
             }
             return options;
         }
-
-        private async Task CopyRemoteObjects_VerifyProperties(
-            ShareClient sourceContainer,
-            ShareClient destinationContainer,
-            TransferPropertiesTestType propertiesType)
-        {
-            // Arrange
-            int size = Constants.KB;
-            string sourcePrefix = "sourceFolder";
-            string destPrefix = "destFolder";
-            await CreateDirectoryInSourceAsync(sourceContainer, sourcePrefix);
-            string itemName1 = string.Join("/", sourcePrefix, GetNewObjectName());
-            string itemName2 = string.Join("/", sourcePrefix, GetNewObjectName());
-            await CreateShareFileAsync(sourceContainer, size, itemName1, propertiesType: propertiesType);
-            await CreateShareFileAsync(sourceContainer, size, itemName2, propertiesType: propertiesType);
-
-            string subDirName = string.Join("/", sourcePrefix, "bar");
-            await CreateDirectoryInSourceAsync(sourceContainer, subDirName);
-            string itemName3 = string.Join("/", subDirName, GetNewObjectName());
-            await CreateShareFileAsync(sourceContainer, size, itemName3, propertiesType: propertiesType);
-
-            string subDirName2 = string.Join("/", sourcePrefix, "pik");
-            await CreateDirectoryInSourceAsync(sourceContainer, subDirName2);
-            string itemName4 = string.Join("/", subDirName2, GetNewObjectName());
-            await CreateShareFileAsync(sourceContainer, size, itemName4, propertiesType: propertiesType);
-
-            await CreateDirectoryInDestinationAsync(destinationContainer, destPrefix);
-
-            // Create storage resource containers
-            StorageResourceContainer sourceResource =
-                GetSourceStorageResourceContainer(sourceContainer, sourcePrefix);
-            StorageResourceContainer destinationResource =
-                GetDestinationStorageResourceContainer(destinationContainer, destPrefix, propertiesType);
-
-            // Create Transfer Manager
-            TransferOptions options = new TransferOptions();
-            TestEventsRaised testEventsRaised = new TestEventsRaised(options);
-            TransferManager transferManager = new TransferManager();
-
-            // Start transfer and await for completion.
-            TransferOperation transfer = await transferManager.StartTransferAsync(
-                sourceResource,
-                destinationResource,
-                options).ConfigureAwait(false);
-
-            CancellationTokenSource tokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30));
-            await TestTransferWithTimeout.WaitForCompletionAsync(
-                transfer,
-                testEventsRaised,
-                tokenSource.Token);
-
-            // Verify completion
-            Assert.NotNull(transfer);
-            Assert.IsTrue(transfer.HasCompleted);
-            Assert.AreEqual(TransferState.Completed, transfer.Status.State);
-            await testEventsRaised.AssertContainerCompletedCheck(4);
-
-            // Assert
-            await VerifyResultsAsync(
-                sourceContainer,
-                sourcePrefix,
-                destinationContainer,
-                destPrefix,
-                propertiesType);
-        }
     }
 }

sdk/storage/Azure.Storage.DataMovement/src/ServiceToServiceJobPart.cs

@@ -3,7 +3,6 @@
 
 using System;
 using System.Collections.Generic;
-using System.Globalization;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
@@ -194,29 +193,26 @@ public override async Task ProcessPartToChunkAsync()
                 }
                 await OnTransferStateChangedAsync(TransferState.InProgress).ConfigureAwait(false);
 
-                long? fileLength = default;
-                StorageResourceItemProperties sourceProperties = default;
-                fileLength = _sourceResource.Length;
-                sourceProperties = await _sourceResource.GetPropertiesAsync(_cancellationToken).ConfigureAwait(false);
-                await _destinationResource.SetPermissionsAsync(
-                    _sourceResource,
-                    sourceProperties,
-                    _cancellationToken).ConfigureAwait(false);
-
-                fileLength = sourceProperties.ResourceLength;
-                if (!fileLength.HasValue)
+                StorageResourceItemProperties sourceProperties =
+                    await _sourceResource.GetPropertiesAsync(_cancellationToken).ConfigureAwait(false);
+                if (!sourceProperties.ResourceLength.HasValue)
                 {
                     await InvokeFailedArgAsync(Errors.UnableToGetLength()).ConfigureAwait(false);
                     return;
                 }
-                long length = fileLength.Value;
+                long length = sourceProperties.ResourceLength.Value;
+
+                await _destinationResource.SetPermissionsAsync(
+                    _sourceResource,
+                    sourceProperties,
+                    _cancellationToken).ConfigureAwait(false);
 
                 // Perform a single copy operation
                 if (_initialTransferSize >= length)
                 {
                     await QueueChunkToChannelAsync(
                         async () =>
-                        await StartSingleCallCopy(length).ConfigureAwait(false))
+                        await StartSingleCallCopy(length, sourceProperties: sourceProperties).ConfigureAwait(false))
                         .ConfigureAwait(false);
                     return;
                 }
@@ -231,7 +227,7 @@ await StartSingleCallCopy(length).ConfigureAwait(false))
                     sourceProperties,
                     _cancellationToken);
 
-                // If we cannot upload in one shot, initiate the parallel block uploader
+                // If we cannot copy in one shot, queue the rest of the chunks
                 if (await CreateDestinationResource(length, blockSize).ConfigureAwait(false))
                 {
                     IEnumerable<(long Offset, long Length)> ranges = GetRanges(length, blockSize);
@@ -261,7 +257,7 @@ await QueueStageBlockRequest(
             }
         }
 
-        internal async Task StartSingleCallCopy(long completeLength)
+        private async Task StartSingleCallCopy(long completeLength, StorageResourceItemProperties sourceProperties)
         {
             try
             {
@@ -276,6 +272,7 @@ await _destinationResource.CopyFromUriAsync(
                     cancellationToken: _cancellationToken).ConfigureAwait(false);
 
                 await ReportBytesWrittenAsync(completeLength).ConfigureAwait(false);
+                await CompleteTransferAsync(sourceProperties).ConfigureAwait(false);
                 await OnTransferStateChangedAsync(TransferState.Completed).ConfigureAwait(false);
             }
             catch (RequestFailedException exception)
@@ -297,11 +294,10 @@ await _destinationResource.CopyFromUriAsync(
         }
 
         /// <summary>
-        /// Creates the destination if necessary, and overwrites if necessary.
-        /// Will return false if no further upload of the blob is necessary
+        /// Creates the destination resource and performs the first copy call.
+        /// Returns false if the copy is complete or this operation failed/skipped.
         /// </summary>
-        /// <returns></returns>
-        internal async Task<bool> CreateDestinationResource(long length, long blockSize)
+        private async Task<bool> CreateDestinationResource(long totalLength, long blockSize)
         {
             try
             {
@@ -311,18 +307,12 @@ await _destinationResource.CopyBlockFromUriAsync(
                     sourceResource: _sourceResource,
                     overwrite: _createMode == StorageResourceCreationMode.OverwriteIfExists,
                     range: new HttpRange(0, blockSize),
-                    completeLength: length,
+                    completeLength: totalLength,
                     options: options,
                     cancellationToken: _cancellationToken).ConfigureAwait(false);
 
                 // Report first chunk written to progress tracker
                 await ReportBytesWrittenAsync(blockSize).ConfigureAwait(false);
-
-                if (blockSize == length)
-                {
-                    await CompleteTransferAsync(options.SourceProperties).ConfigureAwait(false);
-                    return false;
-                }
                 return true;
             }
             catch (RequestFailedException exception)
@@ -335,6 +325,8 @@ await _destinationResource.CopyBlockFromUriAsync(
             {
                 await InvokeFailedArgAsync(ex).ConfigureAwait(false);
             }
+
+            // Do not continue if we need to skip or there was an error.
             return false;
         }