fix(subst): ignore large files

Fixes #9538 This logs a warning for large files (>16MB on 32-bit systems). Signed-off-by: Etienne Millon <[email protected]>
ocaml · Jan 30, 2024 · b3b1b7f · b3b1b7f
1 parent c54da76
commit b3b1b7f
Show file tree

Hide file tree

Showing 3 changed files with 26 additions and 16 deletions.
diff --git a/bin/subst.ml b/bin/subst.ml
@@ -120,15 +120,27 @@ let subst_string s path ~map =
 ;;
 
 let subst_file path ~map =
-  let s = Io.read_file path in
-  let s =
-    if Path.is_root (Path.parent_exn path) && Package.is_opam_file path
-    then "version: \"%%" ^ "VERSION_NUM" ^ "%%\"\n" ^ s
-    else s
-  in
-  match subst_string s ~map path with
-  | None -> ()
-  | Some s -> Io.write_file path s
+  match Io.with_file_in path ~f:Io.read_all_unless_large with
+  | Error () ->
+    let hints =
+      if Sys.word_size = 32
+      then
+        [ Pp.textf
+            "Dune has been built as a 32-bit binary so the maximum size \"dune subst\" \
+             can operate on is 16MiB."
+        ]
+      else []
+    in
+    User_warning.emit ~hints [ Pp.textf "Ignoring large file: %s" (Path.to_string path) ]
+  | Ok s ->
+    let s =
+      if Path.is_root (Path.parent_exn path) && Package.is_opam_file path
+      then "version: \"%%" ^ "VERSION_NUM" ^ "%%\"\n" ^ s
+      else s
+    in
+    (match subst_string s ~map path with
+     | None -> ()
+     | Some s -> Io.write_file path s)
 ;;
 
 (* Extending the Dune_project APIs, but adding capability to modify *)

diff --git a/doc/changes/9811.md b/doc/changes/9811.md
@@ -0,0 +1 @@
+- subst: do not fail on 32-bit systems when large files are encountered. Just log a warning in this case. (#9811, fixes #9538, @emillon)
diff --git a/test/blackbox-tests/test-cases/subst/32bit.t b/test/blackbox-tests/test-cases/subst/32bit.t
@@ -26,10 +26,7 @@ This test uses subst, which needs a git repository:
    create mode 100644 dune-project
    create mode 100644 large.dat
 
-  $ dune subst 2>&1 | head -n 6
-  Error: exception Invalid_argument("Bytes.create")
-  Raised by primitive operation at Stdune__Io.Make.eagerly_input_string in file
-    "otherlibs/stdune/src/io.ml", line 273, characters 14-30
-  Called from Stdune__Io.Make.read_all.(fun) in file
-    "otherlibs/stdune/src/io.ml", line 308, characters 16-40
-  Called from Stdune__Exn.protectx in file "otherlibs/stdune/src/exn.ml", line
+  $ dune subst
+  Warning: Ignoring large file: large.dat
+  Hint: Dune has been built as a 32-bit binary so the maximum size "dune subst"
+  can operate on is 16MiB.
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		- subst: do not fail on 32-bit systems when large files are encountered. Just log a warning in this case. (#9811, fixes #9538, @emillon)