From 2b2b1049795afa1b6449318b5f4caf14176e62d7 Mon Sep 17 00:00:00 2001 From: Joel Dice Date: Mon, 16 May 2022 08:27:36 -0600 Subject: [PATCH] fix: address "too many open files" error when pushing large bindles Pushing a bindle that contains many assets using `StandaloneRead::push` can result in a "too many open files" error on POSIX systems with a default file descriptor limit of 1024. This is due to the unlimited concurrency of `future::join_all`. The alternative is to use `futures::StreamExt::buffer_unordered` instead, which accepts a parameter to limit maximum concurrency. This fixes https://github.com/deislabs/bindle/issues/336. Signed-off-by: Joel Dice --- src/standalone/mod.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/standalone/mod.rs b/src/standalone/mod.rs index 4e54ea7d..889c1c4b 100644 --- a/src/standalone/mod.rs +++ b/src/standalone/mod.rs @@ -4,6 +4,7 @@ use std::convert::TryInto; use std::path::{Path, PathBuf}; use async_compression::tokio::{bufread::GzipDecoder, write::GzipEncoder}; +use futures::{future, stream, TryStreamExt}; use tokio::fs::{read_dir, File}; use tokio::io::{AsyncRead, AsyncWriteExt}; use tokio_stream::{Stream, StreamExt}; @@ -14,6 +15,9 @@ use tracing::{debug, info, instrument, trace}; use crate::client::{tokens::TokenManager, Client, ClientError, Result}; use crate::Id; +/// Maximum number of assets to upload in parallel +const MAX_PARALLEL_UPLOADS: usize = 16; + /// The name of the invoice file pub const INVOICE_FILE: &str = "invoice.toml"; /// The name of the parcels directory @@ -152,11 +156,9 @@ impl StandaloneRead { Ok(()) }); - futures::future::join_all(parcel_futures) + futures::StreamExt::buffer_unordered(stream::iter(parcel_futures), MAX_PARALLEL_UPLOADS) + .try_for_each(future::ok) .await - .into_iter() - .collect::>>()?; - Ok(()) } /// Retrieve the invoice from the standalone bindle.