Merge branch 'main' into adreed/folder-naming

azure-pipelines.yml

@@ -13,9 +13,8 @@ pr:
     include:
       - dev
       - main
-
+     
 jobs:
-
   - job: Build
     timeoutInMinutes: 360
     strategy:
@@ -409,4 +408,107 @@ jobs:
           python ./tool_distributed_mutex.py unlock "$(MUTEX_URL)"
         name: 'Release_the_distributed_mutex'
         # this runs even if the job was canceled (only if the mutex was acquired by this job)
-        condition: and(always(), eq(variables['isMutexSet'], 'true'))
+        condition: and(always(), eq(variables['isMutexSet'], 'true'))
+
+  - job: Additional_E2E_Test_Linux
+    timeoutInMinutes: 360
+    strategy:
+      matrix:
+        Linux:
+          AgentName: "blobfuse-ubuntu20"
+          build_name: 'azcopy_linux_amd64'
+          container_name: 'testcontainer1'
+          GOOS: 'linux'
+          GOARCH: 'amd64'
+          CGO_ENABLED: '0'
+    pool:
+      name: "blobfuse-ubuntu-pool"
+      demands:
+        - ImageOverride -equals $(AgentName)
+
+    steps:
+      - script: |
+          # Install Az.Accounts module using Bash script
+          sudo apt-get update
+          sudo apt-get install -y powershell
+          pwsh -Command "Install-Module -Name Az.Accounts -Scope CurrentUser -Repository PSGallery -AllowClobber -Force"
+        displayName: 'Install PowerShell Az Module'
+
+      - task: GoTool@0
+        inputs:
+          version: $(AZCOPY_GOLANG_VERSION_COVERAGE)
+
+      # Install azcli
+      - script: |
+          curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
+          az --version
+        displayName: 'Install Azure CLI'
+
+      - template: azurePipelineTemplates/run_scenarios.yml
+        parameters:
+          container_name: $(container_name)
+          storage_account_name: $(AZCOPY_E2E_ACCOUNT_NAME)
+          orp_source_account_name: $(ORP_SOURCE_ACCOUNT_NAME)
+          orp_destination_account_name: $(ORP_DESTINATION_ACCOUNT_NAME)
+          orp_source_container_name: $(ORP_SOURCE_CONTAINER_NAME)
+          destination_container_name: $(DESTINATION_CONTAINER_NAME)
+          GOOS: $(GOOS)
+          GOARCH: $(GOARCH)
+          CGO_ENABLED: $(CGO_ENABLED)
+          build_name: $(build_name)
+          azcopy_msi_app_id: $(AZCOPY_MSI_APP_ID)
+
+  - job: Additional_E2E_Test_Windows
+    timeoutInMinutes: 360
+    strategy:
+      matrix:
+        Windows:
+          AgentName: 'azcopy-windows-22'
+          build_name: 'azcopy_windows_amd64.exe'
+          container_name: 'testcontainer2'
+          GOOS: 'windows'
+          GOARCH: 'amd64'
+          CGO_ENABLED: '0'
+    pool:
+      name: "azcopy-windows-pool"
+      demands:
+        - ImageOverride -equals $(AgentName)
+
+    steps:
+      - task: PowerShell@2
+        inputs:
+          targetType: 'inline'
+          script: |
+            # Install Az.Accounts module using PowerShell
+            Install-Module -Name Az.Accounts -Scope CurrentUser -Repository PSGallery -AllowClobber -Force
+          displayName: 'Install PowerShell Az Module'
+
+      - task: GoTool@0
+        inputs:
+          version: $(AZCOPY_GOLANG_VERSION_COVERAGE)
+
+      - task: PowerShell@2
+        displayName: 'Install Azure CLI'
+        inputs:
+          targetType: 'inline'
+          script: |
+            # Install Azure CLI using Chocolatey
+            choco install azure-cli -y
+            az --version
+          
+      - template: azurePipelineTemplates/run_scenarios.yml
+        parameters:
+          container_name: $(container_name)
+          storage_account_name: $(AZCOPY_E2E_ACCOUNT_NAME)
+          orp_source_account_name: $(ORP_SOURCE_ACCOUNT_NAME)
+          orp_destination_account_name: $(ORP_DESTINATION_ACCOUNT_NAME)
+          orp_source_container_name: $(ORP_SOURCE_CONTAINER_NAME)
+          destination_container_name: $(DESTINATION_CONTAINER_NAME)
+          GOOS: $(GOOS)
+          GOARCH: $(GOARCH)
+          CGO_ENABLED: $(CGO_ENABLED)
+          build_name: $(build_name)
+          azcopy_msi_app_id: $(AZCOPY_MSI_APP_ID)
+
+    #TODO: Add MacOS E2E tests after creating macos pool
+

azurePipelineTemplates/run_scenarios.yml

@@ -0,0 +1,106 @@
+parameters:
+  - name: storage_account_name
+    type: string
+  - name: container_name
+    type: string
+  - name: orp_source_account_name
+    type: string
+  - name: orp_destination_account_name
+    type: string
+  - name: orp_source_container_name
+    type: string
+  - name: destination_container_name
+    type: string
+  - name: goos
+    type: string
+  - name: goarch
+    type: string
+  - name: cgo_enabled
+    type: string
+  - name: build_name
+    type: string
+  - name: azcopy_msi_app_id
+    type: string
+
+steps:
+  - task: PowerShell@2
+    displayName: "Run Scenarios"
+    inputs:
+      targetType: 'inline'
+      script: |
+        # Print "Building executable"
+        echo "Building executable"
+        
+        # Set environment variables
+        $build_name = '${{ parameters.build_name }}'
+        $env:GOOS = '${{ parameters.goos }}'
+        $env:GOARCH = '${{ parameters.goarch }}'
+        $env:AZCOPY_AUTO_LOGIN_TYPE = "MSI"
+        $env:AZCOPY_MSI_CLIENT_ID = '${{ parameters.azcopy_msi_app_id }}'
+        $env:CGO_ENABLED = '${{ parameters.cgo_enabled }}'
+
+        go build -cover -o $build_name
+
+        # Check if the executable was created
+        if (-Not (Test-Path "./$build_name")) {
+            Write-Error "Executable not found: $build_name"
+            exit 1
+        }
+        Write-Output "Executable built successfully: $build_name"
+
+        ######################################################## Scenario 1 ########################################################
+
+        # Right now we dont have this fix available in the older version of AzCopy. So, we are using the latest version of AzCopy to run the scenarios.
+        # We will update this workaround once the fix is available in the older version of AzCopy.
+
+        echo "Scenario 1: Test successful copying from BlobToPipe"
+
+        echo "Creating container '${{ parameters.container_name }}' in storage account '${{ parameters.storage_account_name }}'"
+        & "./$build_name" "make" "https://${{ parameters.storage_account_name }}.blob.core.windows.net/${{ parameters.container_name }}"
+
+        echo "Copy a test file from local to the created container"
+        echo "Hello, Azure!" > myfile.txt
+        & "./$build_name" "cp" "myfile.txt" "https://${{ parameters.storage_account_name }}.blob.core.windows.net/${{ parameters.container_name }}"
+
+        & "./$build_name" "cp" "https://${{ parameters.storage_account_name }}.blob.core.windows.net/${{ parameters.container_name }}/myfile.txt" "--from-to=BlobPipe" "--check-md5" "FailIfDifferentOrMissing" > filename
+        
+        if ($LASTEXITCODE -eq 0) {
+            Write-Output "Successfully executed the scenario."
+        } else {
+            Write-Error "AzCopy command failed with exit code $LASTEXITCODE."
+        }
+
+        # Perform cleanup
+        echo "Perform cleanup"
+        echo "Remove the test file from the container"
+        & "./$build_name" "rm" "https://${{ parameters.storage_account_name }}.blob.core.windows.net/${{ parameters.container_name }}/myfile.txt"
+        echo "Remove the container from the storage account"
+        & az storage container delete --name ${{ parameters.container_name }} --account-name ${{ parameters.storage_account_name }}
+
+        ######################################################## Scenario 2 ########################################################
+
+        # Test copying from Object Replication Policy source container to destination container
+        # Assign an object replication policy to a storage account. 
+        # Perform azcopy copy operation to copy the blob to another container.
+
+        echo "Scenario 2: Test copying from Object Replication Policy source container to destination container"
+
+        echo "Creating a test file named 'myfile.txt' and copying it to the source container '${{ parameters.orp_source_container_name }}' within the storage account '${{ parameters.orp_source_account_name }}', which has the ORP policy applied."
+        echo "Hello, Azure!" > myfile.txt
+        & "./$build_name" "cp" "myfile.txt" "https://${{ parameters.orp_source_account_name }}.blob.core.windows.net/${{ parameters.orp_source_container_name }}"
+
+        echo "Copy the test file from the source container to the destination container '${{ parameters.destination_container_name }}' in storage account '${{ parameters.orp_destination_account_name }}'."
+        & "./$build_name" "cp" "https://${{ parameters.orp_source_account_name }}.blob.core.windows.net/${{parameters.orp_source_container_name}}" "https://${{ parameters.orp_destination_account_name }}.blob.core.windows.net/${{parameters.destination_container_name}}" --recursive
+
+        if ($LASTEXITCODE -eq 0) {
+            Write-Output "Successfully executed the scenario."
+        } else {
+            Write-Error "AzCopy command failed with exit code $LASTEXITCODE."
+        }
+
+        # Perform cleanup
+        echo "Perform cleanup"
+        echo "Remove the test file from the source and destination container"
+        & "./$build_name" "rm" "https://${{ parameters.orp_destination_account_name }}.blob.core.windows.net/${{parameters.destination_container_name}}/myfile.txt"
+        & "./$build_name" "rm" "https://${{ parameters.orp_source_account_name }}.blob.core.windows.net/${{ parameters.orp_source_container_name }}/myfile.txt"
+      displayName: "Run Scenarios"

common/hash_data_adapter_hidden_files.go

@@ -35,9 +35,10 @@ func (a *HiddenFileDataAdapter) getHashPath(relativePath string) string {
 	return filepath.Join(basePath, dir, fName)
 }
 
-func (a *HiddenFileDataAdapter) getDataPath(relativePath string) string {
-	return filepath.Join(a.dataBasePath, relativePath)
-}
+// Commenting out this function as it is not used in the codebase
+// func (a *HiddenFileDataAdapter) getDataPath(relativePath string) string {
+// 	return filepath.Join(a.dataBasePath, relativePath)
+// }
 
 func (a *HiddenFileDataAdapter) GetHashData(relativePath string) (*SyncHashData, error) {
 	metaFile := a.getHashPath(relativePath)
@@ -66,9 +67,20 @@ func (a *HiddenFileDataAdapter) SetHashData(relativePath string, data *SyncHashD
 
 	metaFile := a.getHashPath(relativePath)
 
-	f, err := os.OpenFile(metaFile, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0644)
-	if err != nil {
-		return fmt.Errorf("failed to open hash meta file: %w", err)
+	var f *os.File
+	_, err := os.Stat(metaFile)
+	// In windows os.OpenFile function uses the Windows API to manage files, and the combination of O_CREATE, O_TRUNC, and O_RDWR
+	// flags which results in a system call that might not handle hidden files opening operations as expected with this combination of flags.
+	if os.IsNotExist(err) {
+		f, err = os.OpenFile(metaFile, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0644)
+		if err != nil {
+			return fmt.Errorf("failed to create hash meta file: %w", err)
+		}
+	} else {
+		f, err = os.OpenFile(metaFile, os.O_TRUNC|os.O_RDWR, 0644)
+		if err != nil {
+			return fmt.Errorf("failed to open hash meta file: %w", err)
+		}
 	}
 
 	buf, err := json.Marshal(data)
@@ -83,9 +95,9 @@ func (a *HiddenFileDataAdapter) SetHashData(relativePath string, data *SyncHashD
 
 	// Push types around to check for OS-specific hide file method
 	if adapter, canHide := any(a).(interface{ HideFile(string) error }); canHide {
-		dataFile := a.getDataPath(relativePath)
-
-		err := adapter.HideFile(dataFile)
+		// if --local-hash-storage-mode HiddenFiles is used, hide the hash file
+		metaFile := a.getHashPath(relativePath)
+		err := adapter.HideFile(metaFile)
 		if err != nil {
 			return fmt.Errorf("failed to hide file: %w", err)
 		}

e2e-cleanup.ps1

@@ -0,0 +1,30 @@
+Set-PSDebug -Trace 1
+
+$rgs = Get-AzResourceGroup -Name azcopy-newe2e-*
+
+$rmTarget = @()
+
+foreach($rg in $rgs) {
+    if ($rg.Tags)
+    {
+        $unixTime = $rg.Tags["creation"]
+
+        if ($unixTime) {
+            $date = (Get-Date 01.01.1970)+[System.TimeSpan]::FromSeconds($unixTime)
+            if ($date.AddDays(1).CompareTo((Get-Date)) -eq -1) {
+                $rmTarget = $rmTarget + @($rg)
+            }
+        } else {
+            # If it isn't present, it's considered too old and needs to be deleted.
+            $rmTarget = $rmTarget + @($rg)
+        }
+    } else {
+        $rmTarget = $rmTarget + @($rg)
+    }
+}
+
+foreach ($rg in $rmTarget) {
+    $rgn = $rg.ResourceGroupName
+    Write-Output "Removing $rgn"
+    Remove-AzResourceGroup -Name $rgn -Force
+}

e2e-cleanup.yml

@@ -0,0 +1,20 @@
+trigger: none
+pr: none
+
+schedules:
+- cron: "0 0 * * *"
+  displayName: "E2E cleanup (Nightly)"
+  branches:
+    include: [ "main" ]
+  always: true
+
+steps:
+- task: AzurePowerShell@5
+  displayName: Clean up E2E runs
+  inputs:
+    azureSubscription: 'e2ecleanup'
+    ScriptType: 'FilePath'
+    ScriptPath: 'e2e-cleanup.ps1'
+    FailOnStandardError: true
+    pwsh: true
+    azurePowershellVersion: 'LatestVersion'

e2etest/newe2e_arm_resource_group.go

@@ -23,7 +23,7 @@ type ARMResourceGroupCreateParams struct {
 	Location   string                `json:"location"` // required
 	ManagedBy  *string               `json:"managedBy,omitempty"`
 	Properties *ARMResourceGroupInfo `json:"properties,omitempty"`
-	Tags       []string              `json:"tags,omitempty"`
+	Tags       map[string]string     `json:"tags,omitempty"`
 }
 
 type ARMResourceGroupProvisioningStateOutput struct {

e2etest/newe2e_arm_test_hooks.go

@@ -1,9 +1,11 @@
 package e2etest
 
 import (
+	"fmt"
 	"github.com/google/uuid"
 	"net/http"
 	"strings"
+	"time"
 )
 
 var CommonARMClient *ARMClient
@@ -34,6 +36,9 @@ func SetupArmClient(a Asserter) {
 
 	_, err = CommonARMResourceGroup.CreateOrUpdate(ARMResourceGroupCreateParams{
 		Location: "West US", // todo configurable
+		Tags: map[string]string{
+			"creation": fmt.Sprintf("%d", time.Now().UTC().Unix()),
+		},
 	})
 	a.NoError("create resource group", err)
 }

e2etest/newe2e_runazcopy_stdout.go

@@ -11,6 +11,7 @@ var _ AzCopyStdout = &AzCopyParsedStdout{}
 var _ AzCopyStdout = &AzCopyParsedListStdout{}
 var _ AzCopyStdout = &AzCopyParsedCopySyncRemoveStdout{}
 var _ AzCopyStdout = &AzCopyParsedDryrunStdout{}
+var _ AzCopyStdout = &AzCopyParsedJobsListStdout{}
 
 // ManySubscriberChannel is intended to reproduce the effects of .NET's events.
 // This allows us to *partially* answer the question of how we want to handle testing of prompting in the New E2E framework.
@@ -194,3 +195,26 @@ func (a *AzCopyParsedDryrunStdout) Write(p []byte) (n int, err error) {
 
 	return a.AzCopyParsedStdout.Write(p)
 }
+
+type AzCopyParsedJobsListStdout struct {
+	AzCopyParsedStdout
+	listenChan chan<- common.JsonOutputTemplate
+	JobsCount  int
+}
+
+func (a *AzCopyParsedJobsListStdout) Write(p []byte) (n int, err error) {
+	if a.listenChan == nil {
+		a.listenChan = a.OnParsedLine.SubscribeFunc(func(line common.JsonOutputTemplate) {
+			if line.MessageType == common.EOutputMessageType.EndOfJob().String() {
+				var tx common.ListJobsResponse
+				err = json.Unmarshal([]byte(line.MessageContent), &tx)
+				if err != nil {
+					return
+				}
+
+				a.JobsCount = len(tx.JobIDDetails)
+			}
+		})
+	}
+	return a.AzCopyParsedStdout.Write(p)
+}