Support uploading files with Size filtering and AQL avoiding

README.md

@@ -224,7 +224,7 @@
       - [Distribute Release Bundle](#distribute-release-bundle)
       - [Delete Release Bundle Version](#delete-release-bundle-version)
       - [Delete Release Bundle Version Promotion](#delete-release-bundle-version-promotion)
-      - [Export Release Bundle](#export-release-bundle)
+      - [Export Release Bundle Archive](#export-release-bundle-archive)
       - [Import Release Bundle](#import-release-bundle)
       - [Remote Delete Release Bundle](#remote-delete-release-bundle)
   - [Lifecycle APIs](#lifecycle-apis)
@@ -455,7 +455,10 @@ params.ChecksumsCalcEnabled = false
 targetProps := utils.NewProperties()
 targetProps.AddProperty("key1", "val1")
 params.TargetProps = targetProps
-
+// When using the 'archive' option for upload, we can control the target path inside the uploaded archive using placeholders. This operation determines the TargetPathInArchive value.
+TargetPathInArchive := "archive/path/"
+// Size limit for files to be uploaded.
+SizeLimit= &fspatterns.SizeThreshold{SizeInBytes: 10000, Condition: fspatterns.LessThan}
 totalUploaded, totalFailed, err := rtManager.UploadFiles(params)
 ```
 
@@ -484,7 +487,9 @@ params.Retries = 5
 params.SplitCount = 2
 // MinSplitSize default value: 5120
 params.MinSplitSize = 7168
-
+// Optional fields to avoid AQL request
+Sha256 = "5feceb66ffc86f38d952786c6d696c79c2dbc239dd4e91b46729d73a27fb57e9"
+Size = 1000
 totalDownloaded, totalFailed, err := rtManager.DownloadFiles(params)
 ```
 

artifactory/services/download.go

@@ -2,16 +2,16 @@ package services
 
 import (
 	"errors"
+	"github.com/jfrog/build-info-go/entities"
+	biutils "github.com/jfrog/build-info-go/utils"
+	ioutils "github.com/jfrog/gofrog/io"
+	"github.com/jfrog/gofrog/version"
 	"net/http"
 	"os"
 	"path"
 	"path/filepath"
 	"sort"
-
-	biutils "github.com/jfrog/build-info-go/utils"
-	"github.com/jfrog/gofrog/version"
-
-	"github.com/jfrog/build-info-go/entities"
+	"strings"
 
 	"github.com/jfrog/jfrog-client-go/http/httpclient"
 
@@ -164,17 +164,30 @@ func (ds *DownloadService) prepareTasks(producer parallel.Runner, expectedChan c
 					errorsQueue.AddError(err)
 				}
 			}
+
 			var reader *content.ContentReader
 			// Create handler function for the current group.
 			fileHandlerFunc := ds.createFileHandlerFunc(downloadParams, successCounters)
-			// Search items.
-			switch downloadParams.GetSpecType() {
-			case utils.WILDCARD:
-				reader, err = ds.collectFilesUsingWildcardPattern(downloadParams)
-			case utils.BUILD:
-				reader, err = utils.SearchBySpecWithBuild(downloadParams.GetFile(), ds)
-			case utils.AQL:
-				reader, err = utils.SearchBySpecWithAql(downloadParams.GetFile(), ds, utils.SYMLINK)
+			// Check if we can avoid using AQL to get the file's info.
+			avoidAql, err := isFieldsProvidedToAvoidAql(downloadParams)
+			// Check for search errors.
+			if err != nil {
+				log.Error(err)
+				errorsQueue.AddError(err)
+				continue
+			}
+			if avoidAql {
+				reader, err = createResultsItemWithoutAql(downloadParams)
+			} else {
+				// Search items using AQL and get their details (size/checksum/etc.) from Artifactory.
+				switch downloadParams.GetSpecType() {
+				case utils.WILDCARD:
+					reader, err = utils.SearchBySpecWithPattern(downloadParams.GetFile(), ds, utils.SYMLINK)
+				case utils.BUILD:
+					reader, err = utils.SearchBySpecWithBuild(downloadParams.GetFile(), ds)
+				case utils.AQL:
+					reader, err = utils.SearchBySpecWithAql(downloadParams.GetFile(), ds, utils.SYMLINK)
+				}
 			}
 			// Check for search errors.
 			if err != nil {
@@ -196,8 +209,49 @@ func (ds *DownloadService) prepareTasks(producer parallel.Runner, expectedChan c
 	}()
 }
 
-func (ds *DownloadService) collectFilesUsingWildcardPattern(downloadParams DownloadParams) (*content.ContentReader, error) {
-	return utils.SearchBySpecWithPattern(downloadParams.GetFile(), ds, utils.SYMLINK)
+func isFieldsProvidedToAvoidAql(downloadParams DownloadParams) (bool, error) {
+	if downloadParams.Sha256 != "" && downloadParams.Size != nil {
+		// If sha256 and size is provided, we can avoid using AQL to get the file's info.
+		return true, nil
+	} else if downloadParams.Sha256 == "" && downloadParams.Size == nil {
+		// If sha256 and size is missing, we can't avoid using AQL to get the file's info.
+		return false, nil
+	}
+	// If only one of the fields is provided, return an error.
+	return false, errors.New("both sha256 and size must be provided in order to avoid using AQL")
+}
+
+func createResultsItemWithoutAql(downloadParams DownloadParams) (*content.ContentReader, error) {
+	writer, err := content.NewContentWriter(content.DefaultKey, true, false)
+	if err != nil {
+		return nil, err
+	}
+	defer ioutils.Close(writer, &err)
+	repo, path, name, err := breakFileDownloadPathToParts(downloadParams.GetPattern())
+	if err != nil {
+		return nil, err
+	}
+	resultItem := &utils.ResultItem{
+		Type:   string(utils.File),
+		Repo:   repo,
+		Path:   path,
+		Name:   name,
+		Size:   *downloadParams.Size,
+		Sha256: downloadParams.Sha256,
+	}
+	writer.Write(*resultItem)
+	return content.NewContentReader(writer.GetFilePath(), writer.GetArrayKey()), nil
+}
+
+func breakFileDownloadPathToParts(downloadPath string) (repo, path, name string, err error) {
+	if utils.IsWildcardPattern(downloadPath) {
+		return "", "", "", errorutils.CheckErrorf("downloading without AQL is not supported for the provided wildcard pattern: " + downloadPath)
+	}
+	parts := strings.Split(downloadPath, "/")
+	repo = parts[0]
+	path = strings.Join(parts[1:len(parts)-1], "/")
+	name = parts[len(parts)-1]
+	return
 }
 
 func (ds *DownloadService) produceTasks(reader *content.ContentReader, downloadParams DownloadParams, producer parallel.Runner, fileHandler fileHandlerFunc, errorsQueue *clientutils.ErrorsQueue) int {
@@ -399,6 +453,7 @@ func (ds *DownloadService) downloadFile(downloadFileDetails *httpclient.Download
 		LocalFileName:           downloadFileDetails.LocalFileName,
 		LocalPath:               downloadFileDetails.LocalPath,
 		ExpectedSha1:            downloadFileDetails.ExpectedSha1,
+		ExpectedSha256:          downloadFileDetails.ExpectedSha256,
 		FileSize:                downloadFileDetails.Size,
 		SplitCount:              downloadParams.SplitCount,
 		Explode:                 downloadParams.Explode,
@@ -515,8 +570,8 @@ func (ds *DownloadService) createFileHandlerFunc(downloadParams DownloadParams,
 				return err
 			}
 			if downloadParams.IsSymlink() {
-				if isSymlink, e := ds.createSymlinkIfNeeded(ds.GetArtifactoryDetails().GetUrl(), localPath, localFileName, logMsgPrefix, downloadData, successCounters, threadId, downloadParams); isSymlink {
-					return e
+				if isSymlink, err := ds.createSymlinkIfNeeded(ds.GetArtifactoryDetails().GetUrl(), localPath, localFileName, logMsgPrefix, downloadData, successCounters, threadId, downloadParams); isSymlink {
+					return err
 				}
 			}
 			if err = ds.downloadFileIfNeeded(downloadPath, localPath, localFileName, logMsgPrefix, downloadData, downloadParams); err != nil {
@@ -591,6 +646,9 @@ type DownloadParams struct {
 	SplitCount              int
 	PublicGpgKey            string
 	SkipChecksum            bool
+	// Optional fields to avoid AQL request
+	Sha256 string
+	Size   *int64
 }
 
 func (ds *DownloadParams) IsFlat() bool {

artifactory/services/download_test.go

@@ -0,0 +1,38 @@
+package services
+
+import (
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func TestBreakFileDownloadPathToParts(t *testing.T) {
+	testCases := []struct {
+		name         string
+		downloadPath string
+		expectedRepo string
+		expectedPath string
+		expectedName string
+		expectError  bool
+	}{
+		{"Single level path", "repo/file.txt", "repo", "", "file.txt", false},
+		{"Multi-level path", "repo/folder/subfolder/file.txt", "repo", "folder/subfolder", "file.txt", false},
+		{"Root level file", "repo/", "", "", "", true},
+		{"Empty path", "", "", "", "", true},
+		{"Invalid path", "file.txt", "", "", "", true},
+		{"Wildcard path", "repo/*.txt", "", "", "", true},
+	}
+
+	for _, tt := range testCases {
+		t.Run(tt.name, func(t *testing.T) {
+			repo, path, name, err := breakFileDownloadPathToParts(tt.downloadPath)
+			if tt.expectError {
+				assert.Error(t, err)
+			} else {
+				assert.NoError(t, err)
+			}
+			assert.Equal(t, tt.expectedRepo, repo)
+			assert.Equal(t, tt.expectedPath, path)
+			assert.Equal(t, tt.expectedName, name)
+		})
+	}
+}

artifactory/services/fspatterns/sizethreshold.go

@@ -0,0 +1,27 @@
+package fspatterns
+
+// ThresholdCondition represents whether the threshold is for files above or below a specified size.
+type ThresholdCondition int
+
+const (
+	// GreaterThan is greater & equal
+	GreaterEqualThan ThresholdCondition = iota
+	// LessThan is only less
+	LessThan
+)
+
+type SizeThreshold struct {
+	SizeInBytes int64
+	Condition   ThresholdCondition
+}
+
+func (st SizeThreshold) IsSizeWithinThreshold(actualSizeInBytes int64) bool {
+	switch st.Condition {
+	case GreaterEqualThan:
+		return actualSizeInBytes >= st.SizeInBytes
+	case LessThan:
+		return actualSizeInBytes < st.SizeInBytes
+	default:
+		return false
+	}
+}

artifactory/services/fspatterns/sizethreshold_test.go

@@ -0,0 +1,53 @@
+package fspatterns
+
+import (
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func TestSizeWithinLimits(t *testing.T) {
+	tests := []struct {
+		name           string
+		st             SizeThreshold
+		actualSize     int64
+		expectedResult bool
+	}{
+		{
+			name:           "Exact size as threshold and condition is GreaterEqualThan returns true",
+			st:             SizeThreshold{SizeInBytes: 100, Condition: GreaterEqualThan},
+			actualSize:     100,
+			expectedResult: true,
+		},
+		{
+			name:           "SizeInBytes above threshold and condition is GreaterEqualThan returns true",
+			st:             SizeThreshold{SizeInBytes: 100, Condition: GreaterEqualThan},
+			actualSize:     150,
+			expectedResult: true,
+		},
+		{
+			name:           "SizeInBytes below threshold and condition is GreaterEqualThan returns false",
+			st:             SizeThreshold{SizeInBytes: 100, Condition: GreaterEqualThan},
+			actualSize:     50,
+			expectedResult: false,
+		},
+		{
+			name:           "Exact size as threshold and condition is LessThan returns false",
+			st:             SizeThreshold{SizeInBytes: 100, Condition: LessThan},
+			actualSize:     100,
+			expectedResult: false,
+		},
+		{
+			name:           "SizeInBytes above threshold and condition is LessThan returns false",
+			st:             SizeThreshold{SizeInBytes: 100, Condition: LessThan},
+			actualSize:     150,
+			expectedResult: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := tt.st.IsSizeWithinThreshold(tt.actualSize)
+			assert.Equal(t, tt.expectedResult, result)
+		})
+	}
+}