From a808932529dc260a92e876a3a187f61300c5e171 Mon Sep 17 00:00:00 2001 From: Ali Caglayan Date: Mon, 6 Feb 2023 10:16:39 +0100 Subject: [PATCH 1/8] doc: fix cram test in doc (#6995) Signed-off-by: Ali Caglayan --- doc/tests.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/doc/tests.rst b/doc/tests.rst index adee779cfda7..f216e2a352b3 100644 --- a/doc/tests.rst +++ b/doc/tests.rst @@ -612,10 +612,11 @@ access their contents in the test script ``run.t``: .. code:: bash - $ wc -l foo | awk '{ print $1 }' - 4 - $ wc -l $(ls bar) | awk '{ print $1 }' - 1231 + Testing wc: + $ wc -l foo | awk '{ print $1 }' + 4 + $ wc -l $(ls bar) | awk '{ print $1 }' + 1231 Test Options From 14daaf6f0f9b12769c7b07cae7309d4cce5ebc54 Mon Sep 17 00:00:00 2001 From: Alpha Issiaga DIALLO Date: Mon, 6 Feb 2023 11:31:52 +0100 Subject: [PATCH 2/8] dune describe external-lib-deps: printing out more information (#6839) * Print out more information The command print out more information, the package in which an external library belongs to, the libraries, executables and tests at this point with their respective external_lib_deps. It was possible with the command `dune external-lib-deps` which was removed, to use `@install` `@runtest` aliasis. And we ended up with the new command to not be able to use those, this is why knowing the package of an external library could help. The goal is to have much information for `opam-dune-lint`. Signed-off-by: Alpha DIALLO * Clean the code and fix the tests output Signed-off-by: Alpha DIALLO --------- Signed-off-by: Alpha DIALLO Co-authored-by: Etienne Millon --- bin/describe.ml | 187 ++++++++++-------- .../exclude-internal-deps.t/run.t | 12 +- .../external-lib-deps/simple-pps.t/run.t | 29 ++- .../external-lib-deps/simple.t/run.t | 7 +- 4 files changed, 136 insertions(+), 99 deletions(-) diff --git a/bin/describe.ml b/bin/describe.ml index d199888657cd..21683fa9165c 100644 --- a/bin/describe.ml +++ b/bin/describe.ml @@ -593,11 +593,6 @@ module External_lib_deps = struct let to_dyn : t -> Dyn.t = function | Required -> String "required" | Optional -> String "optional" - - let merge x y = - match (x, y) with - | Optional, Optional -> Optional - | _ -> Required end type external_lib_dep = @@ -605,11 +600,45 @@ module External_lib_deps = struct ; kind : Kind.t } - type lib_deps = - { dir : Path.Source.t - ; deps : Lib_dep.t list - ; pps : Preprocess.With_instrumentation.t Preprocess.Per_module.t - } + let external_lib_dep_to_dyn t = + let open Dyn in + List [ String (Lib_name.to_string t.name); Kind.to_dyn t.kind ] + + module Item = struct + module Kind = struct + type t = + | Executables + | Library + | Tests + + let to_string = function + | Executables -> "executables" + | Library -> "library" + | Tests -> "tests" + end + + type t = + { kind : Kind.t + ; dir : Path.Source.t + ; external_deps : external_lib_dep list + ; names : string list + ; package : Package.t option + } + + let to_dyn t = + let open Dyn in + let record = + record + [ ("names", (list string) t.names) + ; ( "package" + , option Package.Name.to_dyn (Option.map ~f:Package.name t.package) + ) + ; ("source_dir", String (Path.Source.to_string t.dir)) + ; ("external_deps", list external_lib_dep_to_dyn t.external_deps) + ] + in + Variant (Kind.to_string t.kind, [ record ]) + end let is_external db name = let open Memo.O in @@ -621,30 +650,6 @@ module External_lib_deps = struct | Installed_private | Public _ | Private _ -> false | Installed -> true) - let libs (context : Context.t) (build_system : Dune_rules.Main.build_system) = - let { Dune_rules.Main.conf; contexts = _; _ } = build_system in - let open Memo.O in - let+ dune_files = - Dune_rules.Dune_load.Dune_files.eval conf.dune_files ~context - in - List.concat_map dune_files ~f:(fun (dune_file : Dune_file.t) -> - List.concat_map dune_file.stanzas ~f:(fun stanza -> - let dir = dune_file.dir in - match stanza with - | Dune_file.Executables exes -> - [ { deps = exes.buildable.libraries - ; dir - ; pps = exes.buildable.preprocess - } - ] - | Dune_file.Library lib -> - [ { deps = lib.buildable.libraries - ; dir - ; pps = lib.buildable.preprocess - } - ] - | _ -> [])) - let external_lib_pps db preprocess = let open Memo.O in let* pps = @@ -666,66 +671,76 @@ module External_lib_deps = struct if is_external then Some { name; kind } else None let external_lib_deps db lib_deps = - Memo.parallel_map lib_deps ~f:(fun { deps; dir; pps } -> - let open Memo.O in - let* libs = - deps - |> Memo.parallel_map ~f:(fun lib -> - match lib with - | Lib_dep.Direct (_, name) | Lib_dep.Re_export (_, name) -> ( - let+ v = external_resolve db name Kind.Required in - match v with - | Some x -> [ x ] - | None -> []) - | Lib_dep.Select select -> - select.choices - |> Memo.parallel_map - ~f:(fun (choice : Lib_dep.Select.Choice.t) -> - Lib_name.Set.to_string_list choice.required - @ Lib_name.Set.to_string_list choice.forbidden - |> Memo.parallel_map ~f:(fun name -> - let name = Lib_name.of_string name in - external_resolve db name Kind.Optional) - >>| List.filter_map ~f:(fun x -> x)) - >>| List.concat) - >>| List.concat - in - let+ pps = external_lib_pps db pps in - (dir, libs @ pps)) - - let libs_to_lib_map libs = - List.fold_left ~init:Lib_name.Map.empty libs ~f:(fun acc_map lib -> - Lib_name.Map.update acc_map lib.name ~f:(fun n -> - match n with - | Some k -> Some (Kind.merge k lib.kind) - | None -> Some lib.kind)) - - let libs_dir_to_map libs_dir = - List.fold_left ~init:Path.Source.Map.empty libs_dir - ~f:(fun acc_map (dir, libs) -> - match Path.Source.Map.find acc_map dir with - | None -> Path.Source.Map.add_exn acc_map dir (libs_to_lib_map libs) - | Some libs_map -> - Path.Source.Map.set acc_map dir - (Lib_name.Map.union libs_map (libs_to_lib_map libs) - ~f:(fun _ k1 k2 -> Some (Kind.merge k1 k2)))) + let open Memo.O in + lib_deps + |> Memo.parallel_map ~f:(fun lib -> + match lib with + | Lib_dep.Direct (_, name) | Lib_dep.Re_export (_, name) -> ( + let+ v = external_resolve db name Kind.Required in + match v with + | Some x -> [ x ] + | None -> []) + | Lib_dep.Select select -> + select.choices + |> Memo.parallel_map ~f:(fun (choice : Lib_dep.Select.Choice.t) -> + Lib_name.Set.to_string_list choice.required + @ Lib_name.Set.to_string_list choice.forbidden + |> Memo.parallel_map ~f:(fun name -> + let name = Lib_name.of_string name in + external_resolve db name Kind.Optional) + >>| List.filter_map ~f:(fun x -> x)) + >>| List.concat) + >>| List.concat + + let external_libs db dir libraries preprocess names package kind = + let open Memo.O in + let open Item in + let* lib_deps = external_lib_deps db libraries in + let+ lib_pps = external_lib_pps db preprocess in + Some { kind; dir; names; package; external_deps = lib_deps @ lib_pps } + + let libs db (context : Context.t) + (build_system : Dune_rules.Main.build_system) = + let { Dune_rules.Main.conf; contexts = _; _ } = build_system in + let open Memo.O in + let* dune_files = + Dune_rules.Dune_load.Dune_files.eval conf.dune_files ~context + in + Memo.parallel_map dune_files ~f:(fun (dune_file : Dune_file.t) -> + Memo.parallel_map dune_file.stanzas ~f:(fun stanza -> + let dir = dune_file.dir in + match stanza with + | Dune_file.Executables exes -> + external_libs db dir exes.buildable.libraries + exes.buildable.preprocess + (List.map exes.names ~f:snd) + exes.package Item.Kind.Executables + | Dune_file.Library lib -> + external_libs db dir lib.buildable.libraries + lib.buildable.preprocess + [ Dune_file.Library.best_name lib |> Lib_name.to_string ] + (Dune_file.Library.package lib) + Item.Kind.Library + | Dune_file.Tests tests -> + external_libs db dir tests.exes.buildable.libraries + tests.exes.buildable.preprocess + (List.map tests.exes.names ~f:snd) + tests.exes.package Item.Kind.Tests + | _ -> Memo.return None) + >>| List.filter_opt) + >>| List.concat let external_resolved_libs setup super_context = let open Memo.O in let context = Super_context.context super_context in let* scope = Scope.DB.find_by_dir context.build_dir in let db = Scope.libs scope in - let* libs = libs context setup in - external_lib_deps db libs + libs db context setup + >>| List.filter ~f:(fun (x : Item.t) -> not (x.external_deps = [])) let to_dyn context_name external_resolved_libs = - Dyn.Tuple - [ Dyn.String context_name - ; external_resolved_libs |> libs_dir_to_map - |> Path.Source.Map.filter ~f:(fun m -> not (Lib_name.Map.is_empty m)) - |> Path.Source.Map.to_dyn (fun libs -> - Lib_name.Map.to_dyn Kind.to_dyn libs) - ] + let open Dyn in + Tuple [ String context_name; list Item.to_dyn external_resolved_libs ] let get setup super_context = let open Memo.O in diff --git a/test/blackbox-tests/test-cases/external-lib-deps/exclude-internal-deps.t/run.t b/test/blackbox-tests/test-cases/external-lib-deps/exclude-internal-deps.t/run.t index 1574fe7bc788..bc82b209d9f3 100644 --- a/test/blackbox-tests/test-cases/external-lib-deps/exclude-internal-deps.t/run.t +++ b/test/blackbox-tests/test-cases/external-lib-deps/exclude-internal-deps.t/run.t @@ -4,5 +4,13 @@ print only the external libraries by dir. $ dune describe external-lib-deps (default - ((. ((a required))) - (lib ((a required))))) + ((library + ((names (foo)) + (package ()) + (source_dir .) + (external_deps ((a required))))) + (library + ((names (inter_lib)) + (package ()) + (source_dir lib) + (external_deps ((a required))))))) diff --git a/test/blackbox-tests/test-cases/external-lib-deps/simple-pps.t/run.t b/test/blackbox-tests/test-cases/external-lib-deps/simple-pps.t/run.t index 5dbf60015500..c41d95832cb6 100644 --- a/test/blackbox-tests/test-cases/external-lib-deps/simple-pps.t/run.t +++ b/test/blackbox-tests/test-cases/external-lib-deps/simple-pps.t/run.t @@ -2,13 +2,22 @@ Expected: To get all required and pps packages $ dune describe external-lib-deps (default - ((. - ((a________ required) - (b________ required) - (c________ required) - (d________ required) - (e________ required) - (f________ required) - (h________ required) - (i________ required) - (j________ required))))) + ((library + ((names (foo)) + (package ()) + (source_dir .) + (external_deps + ((a________ required) + (b________ required) + (c________ required) + (f________ required) + (e________ required) + (d________ required))))) + (executables + ((names (prog)) + (package ()) + (source_dir .) + (external_deps + ((h________ required) + (i________ required) + (j________ required))))))) diff --git a/test/blackbox-tests/test-cases/external-lib-deps/simple.t/run.t b/test/blackbox-tests/test-cases/external-lib-deps/simple.t/run.t index 06b446fe59b8..9f0f4405e625 100644 --- a/test/blackbox-tests/test-cases/external-lib-deps/simple.t/run.t +++ b/test/blackbox-tests/test-cases/external-lib-deps/simple.t/run.t @@ -8,4 +8,9 @@ external library dependencies of a simple project > (libraries base doesnotexist.foo)) > EOF $ dune describe external-lib-deps - (default ((. ((base required) (doesnotexist.foo required))))) + (default + ((library + ((names (dummypkg)) + (package (dummypkg)) + (source_dir .) + (external_deps ((base required) (doesnotexist.foo required))))))) From bc010606803128f40382e73da53238003229e9b4 Mon Sep 17 00:00:00 2001 From: Etienne Millon Date: Mon, 6 Feb 2023 16:07:19 +0100 Subject: [PATCH 3/8] Upgrade ocamlformat to 0.24.1 (#7011) Signed-off-by: Etienne Millon --- .ocamlformat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ocamlformat b/.ocamlformat index 9e29ede9c2f4..b5dbb66c510d 100644 --- a/.ocamlformat +++ b/.ocamlformat @@ -1,4 +1,4 @@ -version=0.21.0 +version=0.24.1 profile=conventional ocaml-version=4.08.0 break-separators=before From 3076e6770c11ce8917f6736abaeff237cee2a136 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Mon, 6 Feb 2023 10:57:59 -0600 Subject: [PATCH 4/8] feature(rules): custom alias for cinaps (#6991) Allow setting the alias used to run cinaps actions. This is done to override the behavior of attaching the cinaps action to both the `cinaps` and `runtest` aliases. Signed-off-by: Rudi Grinberg --- CHANGES.md | 4 ++++ src/dune_rules/cinaps.ml | 23 ++++++++++++++----- .../test-cases/cinaps/custom-alias.t | 21 +++++++++++++++++ 3 files changed, 42 insertions(+), 6 deletions(-) create mode 100644 test/blackbox-tests/test-cases/cinaps/custom-alias.t diff --git a/CHANGES.md b/CHANGES.md index cf899ae38ec7..1c09a7ebf6f9 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -119,6 +119,10 @@ Unreleased - Handle "Too many links" errors when using Dune cache on Windows (#6993, @nojb) +- Allow the `cinaps` stanza to set a custom alias. By default, if the alias is + not set then the cinaps actions will be attached to both `@cinaps` and + `@runtest` (#6988, @rgrinberg) + 3.6.2 (2022-12-21) ------------------ diff --git a/src/dune_rules/cinaps.ml b/src/dune_rules/cinaps.ml index 0960dd1ae9d5..657ef911afea 100644 --- a/src/dune_rules/cinaps.ml +++ b/src/dune_rules/cinaps.ml @@ -8,17 +8,21 @@ type t = ; preprocessor_deps : Dep_conf.t list ; runtime_deps : Dep_conf.t list ; cinaps_version : Syntax.Version.t + ; alias : Alias.Name.t option } let name = "cinaps" +let cinaps_alias = Alias.Name.of_string name + type Stanza.t += T of t let syntax = Dune_lang.Syntax.create ~name ~desc:"the cinaps extension" - [ ((1, 0), `Since (1, 11)); ((1, 1), `Since (3, 5)) ] - -let alias = Alias.make (Alias.Name.of_string name) + [ ((1, 0), `Since (1, 11)) + ; ((1, 1), `Since (3, 5)) + ; ((1, 2), `Since (3, 7)) + ] let decode = let open Dune_lang.Decoder in @@ -33,6 +37,7 @@ let decode = field ~default:[] "runtime_deps" (Dune_lang.Syntax.since syntax (1, 1) >>> repeat Dep_conf.decode) and+ cinaps_version = Dune_lang.Syntax.get_exn syntax + and+ alias = field_o "alias" Alias.Name.decode (* TODO use this field? *) and+ _flags = Ocaml_flags.Spec.decode in { loc @@ -42,6 +47,7 @@ let decode = ; preprocessor_deps ; runtime_deps ; cinaps_version + ; alias }) let () = @@ -160,9 +166,14 @@ let gen_rules sctx t ~dir ~scope = (Path.Build.extend_basename fn ~suffix:".cinaps-corrected")) )) in - let cinaps_alias = alias ~dir in + let cinaps_alias = + Alias.make ~dir @@ Option.value t.alias ~default:cinaps_alias + in let* () = Super_context.add_alias_action sctx ~dir ~loc:(Some loc) cinaps_alias action in - Rules.Produce.Alias.add_deps (Alias.runtest ~dir) - (Action_builder.alias cinaps_alias) + match t.alias with + | Some _ -> Memo.return () + | None -> + Rules.Produce.Alias.add_deps (Alias.runtest ~dir) + (Action_builder.alias cinaps_alias) diff --git a/test/blackbox-tests/test-cases/cinaps/custom-alias.t b/test/blackbox-tests/test-cases/cinaps/custom-alias.t new file mode 100644 index 000000000000..7a74238e3b47 --- /dev/null +++ b/test/blackbox-tests/test-cases/cinaps/custom-alias.t @@ -0,0 +1,21 @@ +Custom alias for the cinaps + + $ cat > dune-project < (lang dune 3.7) + > (using cinaps 1.2) + > EOF + + $ cat > dune < (cinaps + > (files foo.ml) + > (alias foo)) + > EOF + + $ touch foo.ml + + $ dune build @foo --display short + cinaps .cinaps.a7811055/cinaps.ml-gen + ocamlc .cinaps.a7811055/.cinaps.eobjs/byte/dune__exe__Cinaps.{cmi,cmo,cmt} + ocamlopt .cinaps.a7811055/.cinaps.eobjs/native/dune__exe__Cinaps.{cmx,o} + ocamlopt .cinaps.a7811055/cinaps.exe + cinaps alias foo From df834cb516d4045f8fb9cf4fa83912a258df4b9e Mon Sep 17 00:00:00 2001 From: Ali Caglayan Date: Mon, 6 Feb 2023 22:10:25 +0100 Subject: [PATCH 5/8] test: check cram disable doesn't fail silently for cram stanza (#7007) Signed-off-by: Ali Caglayan --- .../cram/cram-setting-in-dune-project.t | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/test/blackbox-tests/test-cases/cram/cram-setting-in-dune-project.t b/test/blackbox-tests/test-cases/cram/cram-setting-in-dune-project.t index 58b8790f1a81..ff3065b7cf95 100644 --- a/test/blackbox-tests/test-cases/cram/cram-setting-in-dune-project.t +++ b/test/blackbox-tests/test-cases/cram/cram-setting-in-dune-project.t @@ -37,3 +37,27 @@ executed: File "run.t", line 1, characters 0-0: Error: Files _build/default/run.t and _build/default/run.t.corrected differ. [1] + +With Dune 3.0 and later, we don't get an error since cram tests are enabled by +default: + + $ cat >dune-project< (lang dune 3.0) + > EOF + + $ dune runtest + File "run.t", line 1, characters 0-0: + Error: Files _build/default/run.t and _build/default/run.t.corrected differ. + [1] + +And if we disable them on purpose, we get an error message: + + $ echo "(cram disable)" >> dune-project + $ dune runtest + File "dune", line 1, characters 0-6: + 1 | (cram) + ^^^^^^ + Error: Cram tests are not enabled in this project. + Hint: You can enable cram tests by adding (cram enable) to your dune-project + file. + [1] From 46a4e2d3480c51df2347c7b7ed1f2859e9fc5f54 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Mon, 6 Feb 2023 23:53:27 -0600 Subject: [PATCH 6/8] test: demonstrate the "misc" section (#7014) The "misc" section isn't currently supported by dune. Probably because it breaks the sandboxing guarantees of switches. Signed-off-by: Rudi Grinberg --- .../test-cases/install/misc-section.t | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 test/blackbox-tests/test-cases/install/misc-section.t diff --git a/test/blackbox-tests/test-cases/install/misc-section.t b/test/blackbox-tests/test-cases/install/misc-section.t new file mode 100644 index 000000000000..de740dc0374e --- /dev/null +++ b/test/blackbox-tests/test-cases/install/misc-section.t @@ -0,0 +1,19 @@ +The misc install section isn't supported: + + $ cat >dune-project < (lang dune 3.7) + > (package + > (name xxx)) + > EOF + + $ cat >dune < (install + > (section misc) + > (files foo)) + > EOF + + $ dune build xxx.install 2>&1 | awk '/Internal error/,/Raised/' + Internal error, please report upstream including the contents of _build/log. + Description: + ("Install.Paths.get", {}) + Raised at Stdune__Code_error.raise in file From c8f392fe9f9a2374a15979220e2f15c0ba32cfbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Ch=C3=A1varri?= Date: Tue, 7 Feb 2023 17:53:16 +0100 Subject: [PATCH 7/8] docs: add rule production docs (#7019) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Javier Chávarri --- doc/dev/rule-production.md | 176 +++++++++++++++++++++++++++++++++++++ 1 file changed, 176 insertions(+) create mode 100644 doc/dev/rule-production.md diff --git a/doc/dev/rule-production.md b/doc/dev/rule-production.md new file mode 100644 index 000000000000..0f740476b191 --- /dev/null +++ b/doc/dev/rule-production.md @@ -0,0 +1,176 @@ +# Rule production + +This document describes how rule production works in Dune. It was originally +written by Jérémie Dimino as part of the +[streaming RFC](https://github.com/ocaml/dune/pull/5251), but moved +into the dev documentation as it provides a great overview on how this part of +Dune works at present. + +## How does rule production works? + +### `Dune_engine.Load_rules` + +The production of rules is driven by the module `Load_rules` in the +`dune_engine` library. This library is the build system core of +Dune. It is meant as a general purpose library for writing build +systems, and the Dune software is built on top of it. In theory, +`dune_engine` shouldn't know about `dune` or `dune-project` +files. However, for historical reason this is not the case yet and +`dune_engine` still knows some things about them. + +As we work on Dune, we expect that `dune_engine` will become more and +more agnostic. Even though it is not completely agnostic, we have +successfully been using it to build Jane Street code base, using the +Jane Street rules on top of this core. So it's already more general +than Dune iself. + +For the purpose of this design doc, we will treat `dune_engine` as a +completely general library that doesn't know about `dune` files. + +The main feature of `Load_rules` is the `Load_rules.load_dir` function: + +```ocaml +val load_dir : dir:Path.t -> Loaded.t Memo.t +``` + +`Loaded.t` represents a "loaded" set of rules for a particular +directory. It can also be thought as a "compiled" set of rules. A +`Loaded.t` contains all the rules in existence that produce targets in +`dir`. For instance, given a `Loaded.t` we can figure out all the +files that would be produced in `dir` if we were building everything +that could be built. While `dir` is a build directory, this also +includes files present in the source tree. This is because +`Load_rules.load_dir` implicitly adds copy rules for all source files +present in the source directory that correspond to the build directory +`dir`. For instance, if `dir` is `_build/default/src`, Dune will +implicitly add rules to copy files in `src` to `_build/default/src`. +Except for rules that have the special `promote` or `fallback` modes. + +This is in fact how Dune evaluates globs during the build. Indeed, +when writing `dune` files we work in an imaginary world where both the +source files and the generated files are present. So when we write +`(deps (glob_files *.txt))`, this `*.txt` denotes both `.txt` files +that are present on disk in the source tree but also as the ones that +can be generated by the build. + +In practice, to evaluate `(glob_files *.txt)` in directory `d`, Dune +calls `Load_rules.load_dir ~dir:d` and filter the list of files that can +be built. Similarly, when Dune needs to build a file +`_build/default/src/x`, it first calls `Load_rules.load_dir` with +`_build/default/src` and then looks up a rule that has `x` has +target in the returned `Loaded.t`. The `Load_rules.load_dir` is +memoised, so it can be called multiple times during the build without +guilt. + +While `Load_rules` is responsible for driving the production of rules, +it is part of `dune_engine` which doesn't know about `dune` files and +doesn't know about OCaml libraries or OCaml compilation in general. So +it is not responsible for actually producing the build rules that +allow to build Dune projects. Instead, `Load_rules` defers the actual +production of rules to a callback that it obtains via +`Build_config`. Inside Dune, this callback is implemented by the +`Gen_rules` module inside the `dune_rules` library. `dune_rules` is +the library that is responsible for parsing, interpreting and +compiling `dune` files down to low-level build rules. + +### `Dune_rules.Gen_rules` + +The entry of `Dune_rules.Gen_rules` is the `gen_rules` function. Its +API looks like: + +```ocaml +val gen_rules : + Build_config.Context_or_install.t -> + dir:Path.Build.t -> + string list -> + Build_config.gen_rules_result Memo.t +``` + +Where `Build_config.gen_rules_result` is, in most cases —when the value +returned is `Build_config.Rules _`—, a "raw" set of rules. Raw in the sense +that there is no overlap checks or any other checks. During the rule +production phase, we merely accumulate a set of rules that is later +processed. The API of `gen_rules` is in fact a bit more complex, but +the above definition is enough for the purpose of this document. + +The first thing `gen_rules` does is analyse the directory it is +given. If the directory corresponds to a source directory with a `dune` +file, `gen_rules` will dispatch the call to the part of `dune_rules` +that parses and interprets the `dune` file. This is the simplest case, +but even in this case there are some things worth mentioning. + +For instance, when compiling an OCaml library dune stores the +artifacts for the library in generated dot-directories. For instance, +the cmi files for library `foo` living in source directory `src` will +end up in `_build/default/src/.foo.objs/byte`. We could produce these +rules when `gen_rules` is called with directory +`_build/default/src/.foo.objs/byte`, however that would spread out the +logic for interpreting `library` stanzas. It is much simpler to +produce all the build rules corresponding to a `library` stanza in one +go. This is what is happening at the moment: when called with +directory `_build/default/src`, `gen_rules` will not only produce +rules for this directory but will also produce rules for +`_build/default/src/.foo.objs/byte` and various other directories. + +`Load_rules` doesn't know anything about this. And in particular, it +doesn't know that it is the `gen_rules` call for directory +`_build/default/src/` that will produce the rules for the dot +subdirectories. When `Load_rules` loads the rules for the +`.../.foo.objs/byte` sub-directory, it simply calls `gen_rules` with +this directory. It is `gen_rules` that "redirects" the call to the +`_build/default/src` directory by calling +`Load_rules.load_dir_and_produce_its_rules`. This function simply +calls `Load_rules.load_dir` and re-emits all the raw rules that were +returned by the corresponding `gen_rules` call. + +This works because `Load_rules.load_dir` accepts the facts that +`gen_rules` produces rules for many directory at once. It simply +filters out the result. But for things to behave well, the unwritten +following invariant must hold: `gen_rules ~dir:d` is allowed to +generate rules for directory `d'` iff `gen_rules ~dir:d'` emits a call +to `Load_rules.load_dir ~dir:d`. + +This scenario happens in a number of cases. All these cases share a +common pattern: the redirections are always to an ancestor +directory. At the moment, there is one exception to this pattern in +the odoc rules, however it is easy to remove. + +Finally, the `copy_files` stanza creates another form of dependency +between directory. In order to calculate the targets produced by +`copy_files`, which needs to be known at rule production time, we need +to evaluate the glob given to `copy_files`. Which requires doing a +call to `Load_rules.load_dir` as previously described. Contrary to the +other form of dependency we just describe, this ones can go from any +directory to any other directory. For instance, the following stanza +in `src/dune`: + +``` +(copy_files foo/*.txt) +``` + +would create a dependency from `_build/default/src` to + `_build_default/src/foo`. + +So in the end, if we were looking at the internal computation graph of +Dune and narrowing it to just the calls to `Load_rules.load_dir`, we +would see a graph with many edges going from a directory to one of its +ancestor. These would mostly be between generated dot-subdirectories +and their first ancestor that has a corresponding directory in the +source tree. Plus a few other arbitrary ones for each `copy_files` +stanza. + +## Directory targets + +Before directory targets, answering the question "what rules produces +file X?" was easy. Dune would just call `Load_rules.load_dir` and +lookup `X` in the result. With directory targets, things are a bit +more complicated. Indeed, `X` might also be produced by a directory +target in an ancestor directory. This means that `Load_rules.load_dir` +now need to look in parent directories as well, which introduce more +dependencies from directories to their parents and can create cycles +because of `copy_files` stanza that create dependencies in the other +direction. + +At a result, some combinations of `copy_files` and directory targets +don't produce the expected result. This is documented in the test +suite. From 06dde72a18766eed8d3b6fc61eb6d990b14089e0 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Tue, 7 Feb 2023 18:38:17 -0600 Subject: [PATCH 8/8] refactor(rules): remove pointless qualification (#7021) Signed-off-by: Rudi Grinberg --- src/dune_rules/expander.ml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dune_rules/expander.ml b/src/dune_rules/expander.ml index 97bb21e26266..f58767aa82ee 100644 --- a/src/dune_rules/expander.ml +++ b/src/dune_rules/expander.ml @@ -141,7 +141,7 @@ let expand_version { scope; _ } ~source s = let libname = Lib_name.of_string s in let pkgname = Lib_name.package_name libname in if not (String.equal (Package.Name.to_string pkgname) s) then - User_error.raise ~loc:source.Dune_lang.Template.Pform.loc + User_error.raise ~loc:source.loc [ Pp.textf "Library names are not allowed in this position. Only package \ names are allowed" @@ -150,7 +150,7 @@ let expand_version { scope; _ } ~source s = Lib.DB.find (Scope.libs scope) libname >>| function | Some lib -> value_from_version (Lib_info.version (Lib.info lib)) | None -> - User_error.raise ~loc:source.Dune_lang.Template.Pform.loc + User_error.raise ~loc:source.loc [ Pp.textf "Package %S doesn't exist in the current project and isn't \ installed either."