feat: add PartSize option to object-storage destination

README.md

@@ -99,6 +99,10 @@ format and keys format.
 
 * [x] Allow multiple public keys per backup
 
+### 0.6.1
+
+* [x] Object storage: allow to customize part size
+
 ### 0.7
 
 * [ ] FTP/SFTP support

destinations/object-storage.go

@@ -23,10 +23,11 @@ var (
 )
 
 type objectStorageDestination struct {
-	options *uback.Options
-	prefix  string
-	bucket  string
-	client  *minio.Client
+	options  *uback.Options
+	prefix   string
+	bucket   string
+	client   *minio.Client
+	partSize uint64
 }
 
 func newObjectStorageDestination(options *uback.Options) (uback.Destination, error) {
@@ -40,6 +41,7 @@ func newObjectStorageDestination(options *uback.Options) (uback.Destination, err
 	accessKeyID := u.User.Username()
 	secretAccessKey, _ := u.User.Password()
 	bucket := u.Path
+	partSize := uint64(0)
 
 	if options.String["Secure"] != "" {
 		s, err := strconv.ParseBool(options.String["Secure"])
@@ -73,6 +75,15 @@ func newObjectStorageDestination(options *uback.Options) (uback.Destination, err
 	}
 	bucket = strings.Trim(bucket, "/")
 
+	if options.String["PartSize"] != "" {
+		ps, err := strconv.ParseUint(options.String["PartSize"], 10, 64)
+		if err != nil {
+			osLog.Warnf("cannot parse PartSize option: %v", err)
+		} else {
+			partSize = ps * 1024 * 1024
+		}
+	}
+
 	client, err := minio.New(endpoint, &minio.Options{
 		Creds:  credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
 		Secure: secure,
@@ -82,7 +93,7 @@ func newObjectStorageDestination(options *uback.Options) (uback.Destination, err
 		return nil, fmt.Errorf("failed to create object storage instance: %v", err)
 	}
 
-	return &objectStorageDestination{options: options, client: client, prefix: prefix, bucket: bucket}, nil
+	return &objectStorageDestination{options: options, client: client, prefix: prefix, bucket: bucket, partSize: partSize}, nil
 }
 
 func (d *objectStorageDestination) ListBackups() ([]uback.Backup, error) {
@@ -129,7 +140,7 @@ func (d *objectStorageDestination) RemoveBackup(backup uback.Backup) error {
 
 func (d *objectStorageDestination) SendBackup(backup uback.Backup, data io.Reader) error {
 	osLog.Printf("writing backup to %s", d.prefix+backup.Filename())
-	_, err := d.client.PutObject(context.Background(), d.bucket, d.prefix+backup.Filename(), data, -1, minio.PutObjectOptions{})
+	_, err := d.client.PutObject(context.Background(), d.bucket, d.prefix+backup.Filename(), data, -1, minio.PutObjectOptions{PartSize: d.partSize})
 	if err != nil {
 		d.client.RemoveObject(context.Background(), d.bucket, d.prefix+backup.Filename(), minio.RemoveObjectOptions{}) //nolint:errcheck
 		return fmt.Errorf("failed to write backup to object storage: %v", err)

doc/dest-object-storage.md

@@ -61,3 +61,24 @@ Optional, defaults to empty.
 
 If you want to host multiple destinations on a single bucket, you have
 to use a different prefix for each destination.
+
+### PartSize
+
+Optional, defaults to 512 MiB (minimum value: 16 MiB).
+
+When streaming backups to object storage, data must be uploaded in parts
+since the total size is unknown in advance. This setting controls the
+size of each part in megabytes (MiB).
+
+Important considerations:
+
+* Storage providers limit uploads to 10,000 parts maximum. This means
+your PartSize setting directly determines the maximum backup size possible
+(PartSize × 10,000).
+
+* Memory usage: A buffer equal to PartSize is allocated in memory. For
+example, the default 512 MiB setting requires 512 MiB of RAM.
+
+* Performance tradeoff: Smaller parts mean more round-trips to the server,
+potentially reducing performance for large backups. Larger parts mean
+fewer round-trips but higher memory usage.