[Storage] Support DatalakeGen2 GA, and upgrade Blob to Track2 SDK

build.proj

@@ -173,7 +173,7 @@
     <!-- Delete powershell runtime files -->
     <PropertyGroup>
       <RuntimeDllsIncludeList>Microsoft.Powershell.*.dll,System*.dll,Microsoft.VisualBasic.dll,Microsoft.CSharp.dll,Microsoft.CodeAnalysis.dll,Microsoft.CodeAnalysis.CSharp.dll</RuntimeDllsIncludeList>
-      <RuntimeDllsExcludeList>System.Security.Cryptography.ProtectedData.dll,System.Configuration.ConfigurationManager.dll,System.Runtime.CompilerServices.Unsafe.dll,System.IO.FileSystem.AccessControl.dll,System.Buffers.dll,System.Text.Encodings.Web.dll,System.CodeDom.dll,System.Management.dll</RuntimeDllsExcludeList>
+      <RuntimeDllsExcludeList>System.Security.Cryptography.ProtectedData.dll,System.Configuration.ConfigurationManager.dll,System.Runtime.CompilerServices.Unsafe.dll,System.IO.FileSystem.AccessControl.dll,System.Buffers.dll,System.Text.Encodings.Web.dll,System.CodeDom.dll,System.Management.dll,System.Text.Json.dll,System.Threading.Tasks.Extensions.dll</RuntimeDllsExcludeList>
     </PropertyGroup>
     <Exec Command="$(PowerShellCoreCommandPrefix) &quot;Get-ChildItem -Path $(RepoArtifacts)/$(Configuration) -Recurse -Include $(RuntimeDllsIncludeList) -Exclude $(RuntimeDllsExcludeList) | Where-Object {$_.FullName -notlike '*PreloadAssemblies*' -and $_.FullName -notlike '*NetCoreAssemblies*'} | Remove-Item -Force&quot;" />
     <Exec Command="$(PowerShellCoreCommandPrefix) &quot;Get-ChildItem -Path $(RepoArtifacts)/$(Configuration) -Recurse -Include 'runtimes' | Remove-Item -Recurse -Force&quot;" Condition="'$(CodeSign)' == 'true'" />

src/Storage/Storage.Management.Test/ScenarioTests/StorageDataPlaneTests.cs

@@ -83,5 +83,12 @@ public void TestCommon()
         {
             TestController.NewInstance.RunPsTest(_logger, $"Test-Common -ResourceGroupName \"{resourceGroupName}\" -StorageAccountName \"{storageAccountName}\"");
         }
+
+        [Fact]
+        [Trait(Category.RunType, Category.LiveOnly)]
+        public void TestDatalakeGen2()
+        {
+            TestController.NewInstance.RunPsTest(_logger, $"Test-DatalakeGen2 -ResourceGroupName \"{resourceGroupName}\" -StorageAccountName \"{storageAccountName}\"");
+        }
     }
 }

src/Storage/Storage.Management.Test/ScenarioTests/StorageDataPlaneTests.ps1

@@ -207,6 +207,26 @@ function Test-Blob
         Assert-AreEqual $blob.Count 2
         Get-AzStorageBlob -Container $containerName -Blob $objectName2 -Context $storageContext | Remove-AzStorageBlob -Force 
 
+		#check XSCL Track2 Items works for container
+		$container = Get-AzStorageContainer $containerName -Context $storageContext
+		$containerProperties = $container.BlobContainerClient.GetProperties().Value
+		Assert-AreEqual $container.BlobContainerProperties.ETag $containerProperties.ETag
+		Set-AzStorageContainerAcl $containerName -Context $storageContext -Permission Blob
+		$containerProperties = $container.BlobContainerClient.GetProperties().Value
+		Assert-AreNotEqual $container.BlobContainerProperties.ETag $containerProperties.ETag
+		$container.FetchAttributes()
+		Assert-AreEqual $container.BlobContainerProperties.ETag $containerProperties.ETag
+
+		#check XSCL Track2 Items works for Blob
+		$blob = Get-AzStorageBlob -Container $containerName -Blob $objectName1 -Context $storageContext
+		$blobProperties = $blob.BlobClient.GetProperties().Value
+		Assert-AreEqual $blob.BlobProperties.ETag $blobProperties.ETag
+		Set-AzStorageBlobContent -File $localSrcFile -Container $containerName -Blob $objectName1 -Force -Context $storageContext
+		$blobProperties = $blob.BlobClient.GetProperties().Value
+		Assert-AreNotEqual $blob.BlobProperties.ETag $blobProperties.ETag
+		$blob.FetchAttributes()
+		Assert-AreEqual $blob.BlobProperties.ETag $blobProperties.ETag
+
         # Copy blob to the same container, but with a different name.
         Start-AzStorageBlobCopy -srcContainer $containerName -SrcBlob $objectName1 -DestContainer $containerName -DestBlob $objectName2 -StandardBlobTier $StandardBlobTier -RehydratePriority High -Context $storageContext -DestContext $storageContext
         Get-AzStorageBlobCopyState -Container $containerName -Blob $objectName2 -Context $storageContext
@@ -684,6 +704,153 @@ function Test-Common
     }    
 }
 
+<#
+    .SYNOPSIS
+    Tests DatalakeGen-only related commands.
+#>
+function Test-DatalakeGen2
+{
+    Param(
+        [Parameter(Mandatory = $True)]
+        [string]
+        $StorageAccountName,
+        [Parameter(Mandatory = $True)]
+        [string]
+        $ResourceGroupName
+    ) 
+
+    New-TestResourceGroupAndStorageAccount -ResourceGroupName $ResourceGroupName -StorageAccountName $StorageAccountName -EnableHNFS $true
+
+    try{
+
+        $storageAccountKeyValue = $(Get-AzStorageAccountKey -ResourceGroupName $ResourceGroupName -Name $StorageAccountName)[0].Value
+        $storageContext = New-AzStorageContext -StorageAccountName $StorageAccountName -StorageAccountKey $storageAccountKeyValue
+
+        $localSrcFile = "localsrcDatalakeGen2testfile.psd1" #The file need exist before test, and should be 512 bytes aligned
+        New-Item $localSrcFile -ItemType File -Force
+        $localDestFile = "localdestDatalakeGen2testfile.txt"
+
+        $filesystemName = "adlsgen2testfilesystem" 
+		$directoryPath1 = "dir1"
+		$directoryPath2 = "dir2"
+		$directoryPath3 = "dir3"
+        $filePath1 = "dir1/Item1.txt"
+        $filePath2 = "dir2/Item2.txt"
+        $filePath3 = "dir2/Item3.txt"
+        $ContentType = "image/jpeg"
+        $ContentMD5 = "i727sP7HigloQDsqadNLHw=="
+
+        # Create FileSystem (actually a container)
+        New-AzDatalakeGen2FileSystem $filesystemName -Context $storageContext
+
+		# Create folders
+		$dir1 = New-AzDataLakeGen2Item -Context $storageContext -FileSystem $filesystemName -Path $directoryPath1 -Directory -Permission rwxrwxrwx -Umask ---rwx---  -Property @{"ContentEncoding" = "UDF8"; "CacheControl" = "READ"} -Metadata  @{"tag1" = "value1"; "tag2" = "value2" }
+		Assert-AreEqual $dir1.Path $directoryPath1
+        Assert-AreEqual $dir1.Permissions.ToSymbolicPermissions() "rwx---rwx"
+		$dir2 = New-AzDataLakeGen2Item -Context $storageContext -FileSystem $filesystemName -Path $directoryPath2 -Directory
+
+		# Create (upload) File
+		$t = New-AzDataLakeGen2Item -Context $storageContext -FileSystem $filesystemName -Path $filePath1 -Source $localSrcFile -Force -AsJob
+		$t | wait-job
+        Assert-AreEqual $t.State "Completed"
+        Assert-AreEqual $t.Error $null
+		$file2 = New-AzDataLakeGen2Item -Context $storageContext -FileSystem $filesystemName -Path $filePath2 -Source $localSrcFile -Permission rwxrwxrwx -Umask ---rwx--- -Property @{"ContentType" = $ContentType; "ContentMD5" = $ContentMD5}  -Metadata  @{"tag1" = "value1"; "tag2" = "value2" }
+        Assert-AreEqual $file2.Path $filePath2
+        Assert-AreEqual $file2.Properties.ContentType $ContentType
+        Assert-AreEqual $file2.Properties.Metadata.Count 2
+        Assert-AreEqual $file2.Permissions.ToSymbolicPermissions() "rwx---rwx"
+
+		# update Blob and Directory
+        $ContentType = "application/octet-stream"
+        $ContentMD5 = "NW/H9Zxr2md6L1/yhNKdew=="
+		$ContentEncoding = "UDF8"
+		## create ACL with 3 ACEs
+		$acl = New-AzDataLakeGen2ItemAclObject -AccessControlType user -Permission rw- 
+		$acl = New-AzDataLakeGen2ItemAclObject -AccessControlType group -Permission rw- -InputObject $acl 
+		$acl = New-AzDataLakeGen2ItemAclObject -AccessControlType other -Permission "-wx" -InputObject $acl
+		##Update File with pipeline		
+		$file1 = Get-AzDataLakeGen2Item -Context $storageContext -FileSystem $filesystemName -Path $filePath1 | Update-AzDataLakeGen2Item  `
+                -Acl $acl `
+                -Property @{"ContentType" = $ContentType; "ContentMD5" = $ContentMD5} `
+                -Metadata  @{"tag1" = "value1"; "tag2" = "value2" } `
+                -Permission rw-rw--wx `
+                -Owner '$superuser' `
+                -Group '$superuser'
+		$file1 = Get-AzDataLakeGen2Item -Context $storageContext -FileSystem $filesystemName -Path $filePath1
+		Assert-AreEqual $file1.Path $filePath1
+        Assert-AreEqual $file1.Permissions.ToSymbolicPermissions() "rw-rw--wx"
+        Assert-AreEqual $file1.Properties.ContentType $ContentType
+        Assert-AreEqual $file1.Properties.Metadata.Count 2
+        Assert-AreEqual $file1.Owner '$superuser'
+        Assert-AreEqual $file1.Group '$superuser'
+		## Update Directory
+		$dir1 = Update-AzDataLakeGen2Item -Context $storageContext -FileSystem $filesystemName -Path $directoryPath1 `
+                 -Acl $acl `
+                 -Property @{"ContentEncoding" = $ContentEncoding} `
+                 -Metadata  @{"tag1" = "value1"; "tag2" = "value2" } `
+                 -Permission rw-rw--wx `
+                 -Owner '$superuser' `
+                 -Group '$superuser' 
+		$dir1 = Get-AzDataLakeGen2Item -Context $storageContext -FileSystem $filesystemName -Path $directoryPath1
+		Assert-AreEqual $dir1.Path $directoryPath1
+        Assert-AreEqual $dir1.Permissions.ToSymbolicPermissions() "rw-rw--wx"
+        Assert-AreEqual $dir1.Properties.ContentEncoding $ContentEncoding
+        Assert-AreEqual $dir1.Properties.Metadata.Count 3  # inlucde "hdi_isfolder" which is handle by server
+        Assert-AreEqual $dir1.Owner '$superuser'
+        Assert-AreEqual $dir1.Group '$superuser'
+
+		#list Items
+		## List direct Items from FileSystem
+		$items = Get-AzDataLakeGen2ChildItem -Context $storageContext -FileSystem $filesystemName -FetchPermission
+        Assert-AreEqual $items.Count 2
+		Assert-NotNull $items[0].Permissions
+		$items = Get-AzDataLakeGen2ChildItem -Context $storageContext -FileSystem $filesystemName -Recurse 
+        Assert-AreEqual $items.Count 4
+		Assert-Null $items[0].Permissions
+
+		#download File
+		$t  = Get-AzDataLakeGen2ItemContent -Context $storageContext -FileSystem $filesystemName -Path $filePath1 -Destination $localDestFile -AsJob -Force
+		$t  | Wait-Job
+		Assert-AreEqual $t.State "Completed"
+		Assert-AreEqual $t.Error $null
+        Assert-AreEqual (Get-FileHash -Path $localDestFile -Algorithm MD5).Hash (Get-FileHash -Path $localSrcFile -Algorithm MD5).Hash
+
+        # Move Items
+		## Move File
+        $file3 = Move-AzDataLakeGen2Item -Context $storageContext -FileSystem $filesystemName -Path $filePath2 -DestFileSystem $filesystemName -DestPath $filePath3 -Force
+		$file3 = Get-AzDataLakeGen2Item -Context $storageContext -FileSystem $filesystemName -Path $filePath3
+		Assert-AreEqual $file3.Path $filePath3
+        Assert-AreEqual $file3.Permissions $file2.Permissions
+		$file2 = $file3 | Move-AzDataLakeGen2Item -DestFileSystem $filesystemName -DestPath $filePath2
+		$file2 = Get-AzDataLakeGen2Item -Context $storageContext -FileSystem $filesystemName -Path $filePath2
+		Assert-AreEqual $file2.Path $filePath2
+        Assert-AreEqual $file2.Permissions $file3.Permissions
+		## Move Folder
+        $dir3 = Move-AzDataLakeGen2Item -Context $storageContext -FileSystem $filesystemName -Path $directoryPath1 -DestFileSystem $filesystemName -DestPath $directoryPath3 
+		$dir3 = Get-AzDataLakeGen2Item -Context $storageContext -FileSystem $filesystemName -Path $directoryPath3
+		Assert-AreEqual $dir3.Path $directoryPath3
+        Assert-AreEqual $dir3.Permissions $dir1.Permissions
+		$dir1 = $dir3 | Move-AzDataLakeGen2Item -DestFileSystem $filesystemName -DestPath $directoryPath1
+		$dir1 = Get-AzDataLakeGen2Item -Context $storageContext -FileSystem $filesystemName -Path $directoryPath1
+		Assert-AreEqual $dir1.Path $directoryPath1
+
+		# Remove Items
+        Remove-AzDataLakeGen2Item -Context $storageContext -FileSystem $filesystemName -Path $filePath1 -Force
+        Remove-AzDataLakeGen2Item -Context $storageContext -FileSystem $filesystemName -Path $directoryPath1 -Force
+
+        # Clean Storage Account
+        Get-AzDataLakeGen2ChildItem -Context $storageContext -FileSystem $filesystemName | Remove-AzDataLakeGen2Item -Force
+
+        # remove File system (actually a container)
+        Remove-AzDatalakeGen2FileSystem $filesystemName -Context $storageContext
+
+    }
+    finally
+    {
+        Clean-ResourceGroup $ResourceGroupName
+    }
+}
+
 function New-TestResourceGroupAndStorageAccount
 { 
     Param(

src/Storage/Storage.Management.Test/Storage.Management.Test.csproj

@@ -11,6 +11,8 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="Azure.Storage.Blobs" Version="12.4.0" />
+    <PackageReference Include="Azure.Storage.Files.DataLake" Version="12.0.0" />
     <PackageReference Include="Microsoft.Azure.Management.Storage" Version="14.5.0" />
   </ItemGroup>
 

src/Storage/Storage.Management/Az.Storage.psd1

@@ -66,7 +66,8 @@ RequiredAssemblies = 'Microsoft.Azure.Management.Storage.dll',
                'Microsoft.Azure.Storage.DataMovement.dll', 
                'Microsoft.Azure.DocumentDB.Core.dll', 'Microsoft.OData.Core.dll', 
                'Microsoft.OData.Edm.dll', 'Microsoft.Spatial.dll', 
-               'Microsoft.Azure.KeyVault.Core.dll'
+               'Microsoft.Azure.KeyVault.Core.dll','Azure.Storage.Blobs.dll',
+               'Azure.Storage.Common.dll', 'Azure.Storage.Files.DataLake.dll'
 
 # Script files (.ps1) that are run in the caller's environment prior to importing this module.
 # ScriptsToProcess = @()
@@ -158,15 +159,22 @@ CmdletsToExport = 'Get-AzStorageAccount', 'Get-AzStorageAccountKey',
                'Revoke-AzStorageAccountUserDelegationKeys', 
                'Get-AzStorageFileHandle', 'Close-AzStorageFileHandle', 
                'New-AzRmStorageShare', 'Remove-AzRmStorageShare', 
-               'Get-AzRmStorageShare', 'Update-AzRmStorageShare'
+               'Get-AzRmStorageShare', 'Update-AzRmStorageShare', 
+               'Get-AzDataLakeGen2ChildItem', 'Get-AzDataLakeGen2Item', 
+               'New-AzDataLakeGen2Item', 'Move-AzDataLakeGen2Item', 
+               'Remove-AzDataLakeGen2Item', 'Update-AzDataLakeGen2Item', 
+               'Set-AzDataLakeGen2ItemAclObject', 'Get-AzDataLakeGen2ItemContent'
+
 
 # Variables to export from this module
 # VariablesToExport = @()
 
 # Aliases to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no aliases to export.
 AliasesToExport = 'Get-AzStorageContainerAcl', 'Start-CopyAzureStorageBlob', 
                'Stop-CopyAzureStorageBlob', 'Enable-AzStorageSoftDelete', 
-               'Disable-AzStorageSoftDelete'
+               'Disable-AzStorageSoftDelete', 
+               'New-AzDatalakeGen2FileSystem', 'Remove-AzDatalakeGen2FileSystem', 
+               'Get-AzDatalakeGen2FileSystem', 'New-AzDataLakeGen2ItemAclObject'
 
 # DSC resources to export from this module
 # DscResourcesToExport = @()

src/Storage/Storage.Management/ChangeLog.md

@@ -22,6 +22,15 @@
 * Supported new SkuName StandardGZRS, StandardRAGZRS when create/update Storage account
     - `New-AzStorageAccount`
     - `Set-AzStorageAccount`
+* Support DataLake Gen2 
+    -  New-AzDataLakeGen2Item
+    -  Get-AzDataLakeGen2Item
+    -  Get-AzDataLakeGen2ChildItem
+    -  Move-AzDataLakeGen2Item
+    -  Set-AzDataLakeGen2ItemAclObject
+    -  Update-AzDataLakeGen2Item
+    -  Get-AzDataLakeGen2ItemContent
+    -  Remove-AzDataLakeGen2Item
 
 ## Version 1.13.0
 * Supported AllowProtectedAppendWrite in ImmutabilityPolicy

src/Storage/Storage.Management/Storage.Management.csproj

@@ -15,11 +15,12 @@
   </PropertyGroup>
   <ItemGroup>
     <PackageReference Include="Microsoft.Azure.Management.Storage" Version="14.5.0" />
-    <PackageReference Include="Microsoft.Azure.Storage.Blob" Version="11.1.0" />
-    <PackageReference Include="Microsoft.Azure.Storage.File" Version="11.1.0" />
-    <PackageReference Include="Microsoft.Azure.Storage.Queue" Version="11.1.0" />
+    <PackageReference Include="Microsoft.Azure.Storage.Blob" Version="11.1.1" />
+    <PackageReference Include="Microsoft.Azure.Storage.File" Version="11.1.1" />
+    <PackageReference Include="Microsoft.Azure.Storage.Queue" Version="11.1.1" />
   </ItemGroup>
-  <ItemGroup>
+
+<ItemGroup>
     <Compile Include="$(StorageToolsPath)Adapters\ARM.Storage.6\ARMStorageProvider.cs" Link="Common\ARMStorageProvider.cs" />
     <Compile Include="$(StorageToolsPath)Adapters\ARM.Storage.6\ARMStorageService.cs" Link="Common\ARMStorageService.cs" />
   </ItemGroup>

src/Storage/Storage.Management/help/Az.Storage.md

@@ -41,6 +41,15 @@ Enable delete retention policy  for the Azure Storage Blob service.
 ### [Enable-AzStorageStaticWebsite](Enable-AzStorageStaticWebsite.md)
 Enable static website for the Azure Storage account.
 
+### [Get-AzDataLakeGen2ChildItem](Get-AzDataLakeGen2ChildItem.md)
+Lists sub directorys and files from a directory or filesystem root.
+
+### [Get-AzDataLakeGen2Item](Get-AzDataLakeGen2Item.md)
+Gets the details of a file or directory in a filesystem.
+
+### [Get-AzDataLakeGen2ItemContent](Get-AzDataLakeGen2ItemContent.md)
+Download a file.
+
 ### [Get-AzRmStorageContainer](Get-AzRmStorageContainer.md)
 Gets or lists Storage blob containers
 
@@ -131,6 +140,12 @@ Gets the Storage resource usage of the current subscription.
 ### [Lock-AzRmStorageContainerImmutabilityPolicy](Lock-AzRmStorageContainerImmutabilityPolicy.md)
 Locks ImmutabilityPolicy of a Storage blob containers
 
+### [Move-AzDataLakeGen2Item](Move-AzDataLakeGen2Item.md)
+Move a file or directory to another a file or directory in same Storage account.
+
+### [New-AzDataLakeGen2Item](New-AzDataLakeGen2Item.md)
+Create a file or directory in a filesystem.
+
 ### [New-AzRmStorageContainer](New-AzRmStorageContainer.md)
 Creates a Storage blob container
 
@@ -200,6 +215,9 @@ Generates an SAS token for an Azure Storage table.
 ### [New-AzStorageTableStoredAccessPolicy](New-AzStorageTableStoredAccessPolicy.md)
 Creates a stored access policy for an Azure storage table.
 
+### [Remove-AzDataLakeGen2Item](Remove-AzDataLakeGen2Item.md)
+Remove a file or directory.
+
 ### [Remove-AzRmStorageContainer](Remove-AzRmStorageContainer.md)
 Removes a Storage blob container
 
@@ -263,6 +281,9 @@ Revoke all User Delegation keys of a Storage account.
 ### [Set-AzCurrentStorageAccount](Set-AzCurrentStorageAccount.md)
 Modifies the current Storage account of the specified subscription.
 
+### [Set-AzDataLakeGen2ItemAclObject](Set-AzDataLakeGen2ItemAclObject.md)
+Creates/Updates a DataLake gen2 item ACL object, which can be used in Update-AzDataLakeGen2Item cmdlet.
+
 ### [Set-AzRmStorageContainerImmutabilityPolicy](Set-AzRmStorageContainerImmutabilityPolicy.md)
 Creates or updates ImmutabilityPolicy of a Storage blob containers
 
@@ -320,6 +341,9 @@ Stops a copy operation.
 ### [Stop-AzStorageFileCopy](Stop-AzStorageFileCopy.md)
 Stops a copy operation to the specified destination file.
 
+### [Update-AzDataLakeGen2Item](Update-AzDataLakeGen2Item.md)
+Update a file or directory on properties, metadata, permission, ACL, and owner.
+
 ### [Update-AzRmStorageContainer](Update-AzRmStorageContainer.md)
 Modifies a Storage blob container