Fix possible deadlock if max_open_files set too low

We now close the file permit if the write future takes
too long.

fixes #907

.bazelrc

@@ -32,7 +32,7 @@ build:debug -c dbg
 build:self_test --remote_instance_name=main
 build:self_test --remote_cache=grpc://127.0.0.1:50051
 
-build:self_execute --remote_executor=grpc://127.0.0.1:50052
+build:self_execute --remote_executor=grpc://127.0.0.1:50051
 build:self_execute --remote_instance_name=main
 build:self_execute --remote_default_exec_properties=cpu_count=1
 build:self_execute --platform_suffix=self-execute

BUILD.bazel

@@ -25,7 +25,6 @@ rust_binary(
         "@crates//:async-lock",
         "@crates//:axum",
         "@crates//:clap",
-        "@crates//:console-subscriber",
         "@crates//:futures",
         "@crates//:hyper",
         "@crates//:mimalloc",
@@ -39,7 +38,6 @@ rust_binary(
         "@crates//:tonic",
         "@crates//:tower",
         "@crates//:tracing",
-        "@crates//:tracing-subscriber",
     ],
 )
 

Cargo.toml

@@ -39,7 +39,6 @@ async-lock = "3.3.0"
 axum = "0.6.20"
 
 clap = { version = "4.5.4", features = ["derive"] }
-console-subscriber = { version = "0.2.0" }
 futures = "0.3.30"
 hyper = { version = "0.14.28" }
 mimalloc = "0.1.41"
@@ -53,4 +52,3 @@ tokio-rustls = "0.25.0"
 tonic = { version = "0.11.0", features = ["gzip", "tls"] }
 tower = "0.4.13"
 tracing = "0.1.40"
-tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }

nativelink-macro/src/lib.rs

@@ -34,6 +34,11 @@ pub fn nativelink_test(attr: TokenStream, item: TokenStream) -> TokenStream {
         #[allow(clippy::disallowed_methods)]
         #[tokio::test(#attr)]
         async fn #fn_name(#fn_inputs) #fn_output {
+            // Error means already initialized, which is ok.
+            let _ = nativelink_util::init_tracing();
+            // If already set it's ok.
+            let _ = nativelink_util::fs::set_idle_file_descriptor_timeout(std::time::Duration::from_millis(100));
+
             #[warn(clippy::disallowed_methods)]
             ::std::sync::Arc::new(::nativelink_util::origin_context::OriginContext::new()).wrap_async(
                 ::nativelink_util::__tracing::trace_span!("test"), async move {

nativelink-store/BUILD.bazel

@@ -116,10 +116,12 @@ rust_test_suite(
         "@crates//:hyper",
         "@crates//:memory-stats",
         "@crates//:once_cell",
+        "@crates//:parking_lot",
         "@crates//:pretty_assertions",
         "@crates//:rand",
         "@crates//:redis",
         "@crates//:redis-test",
+        "@crates//:serial_test",
         "@crates//:sha2",
         "@crates//:tokio",
         "@crates//:tokio-stream",

nativelink-store/Cargo.toml

@@ -53,3 +53,4 @@ aws-smithy-types = "1.1.8"
 aws-sdk-s3 = { version = "1.24.0"  }
 aws-smithy-runtime = { version = "1.3.1", features = ["test-util"] }
 aws-smithy-runtime-api = "1.4.0"
+serial_test = { version = "3.1.1", features = ["async"] }

nativelink-store/src/filesystem_store.rs

@@ -802,6 +802,7 @@ impl<Fe: FileEntry> Store for FilesystemStore<Fe> {
                 digest,
             }),
         );
+        drop(file);
         self.emplace_file(digest, Arc::new(entry))
             .await
             .err_tip(|| "Could not move file into store in upload_file_to_store, maybe dest is on different volume?")?;