From a2f83c601cf1b91af30bc36c8a440691d528cdbc Mon Sep 17 00:00:00 2001 From: Antonio Nuno Monteiro Date: Thu, 24 Nov 2022 19:42:04 -0800 Subject: [PATCH 01/20] refactor(melange): remove bs specific flag (#6571) Signed-off-by: Antonio Nuno Monteiro Co-authored-by: Rudi Grinberg --- Makefile | 3 +- flake.lock | 126 +++++++++++++++++++-------- flake.nix | 8 +- src/dune_rules/module_compilation.ml | 10 +-- 4 files changed, 102 insertions(+), 45 deletions(-) diff --git a/Makefile b/Makefile index f4cd9c465cb..9da2ecfbb4e 100644 --- a/Makefile +++ b/Makefile @@ -74,7 +74,8 @@ dev-depext: .PHONY: melange melange: - opam pin add melange https://github.com/melange-re/melange.git#2f7a184400fd5d62c9160528a7ab4ce81874c024 + opam pin add melange-compiler-libs https://github.com/melange-re/melange-compiler-libs.git#426463a77d0b70ecf0108c98e6a86d325cd01472 + opam pin add melange https://github.com/melange-re/melange.git#685e546e290d317a884a4d48c7835467422c6426 dev-deps: melange opam install -y $(TEST_DEPS) diff --git a/flake.lock b/flake.lock index 0790e84ac61..3eca45f69fd 100644 --- a/flake.lock +++ b/flake.lock @@ -79,10 +79,12 @@ "all-cabal-json": "all-cabal-json", "crane": "crane", "devshell": "devshell", + "flake-parts": "flake-parts", "flake-utils-pre-commit": "flake-utils-pre-commit", "ghc-utils": "ghc-utils", "gomod2nix": "gomod2nix", "mach-nix": "mach-nix", + "nix-pypi-fetcher": "nix-pypi-fetcher", "nixpkgs": [ "melange", "nixpkgs" @@ -91,11 +93,11 @@ "pre-commit-hooks": "pre-commit-hooks" }, "locked": { - "lastModified": 1668851003, - "narHash": "sha256-X7RCQQynbxStZR2m7HW38r/msMQwVl3afD6UXOCtvx4=", + "lastModified": 1669202765, + "narHash": "sha256-j2kCOzlpNHVmwE7Hjb2Fgk7W8DFZ/xXZgsvxSo+CWjU=", "owner": "nix-community", "repo": "dream2nix", - "rev": "c77e8379d8fe01213ba072e40946cbfb7b58e628", + "rev": "861e46a2057aabcdccf7ac81083b2ceb1aa23ed6", "type": "github" }, "original": { @@ -160,6 +162,24 @@ "type": "github" } }, + "flake-parts": { + "inputs": { + "nixpkgs-lib": "nixpkgs-lib" + }, + "locked": { + "lastModified": 1668450977, + "narHash": "sha256-cfLhMhnvXn6x1vPm+Jow3RiFAUSCw/l1utktCw5rVA4=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "d591857e9d7dd9ddbfba0ea02b43b927c3c0f1fa", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, "flake-utils": { "locked": { "lastModified": 1667395993, @@ -337,11 +357,11 @@ "nixpkgs": "nixpkgs" }, "locked": { - "lastModified": 1669089290, - "narHash": "sha256-txvljRihLAqEk/oOKkLKWyPBvmKp8ooncitS7FlH6rc=", + "lastModified": 1669334284, + "narHash": "sha256-ZmHZH3xsPqNQvuZB5BwJf0kPQY2bWrmcTsmuMOLhi5A=", "owner": "melange-re", "repo": "melange", - "rev": "2abacfa6515c034b3780df8d4d2029fcffc90035", + "rev": "685e546e290d317a884a4d48c7835467422c6426", "type": "github" }, "original": { @@ -362,11 +382,11 @@ ] }, "locked": { - "lastModified": 1668541767, - "narHash": "sha256-FzNuf4AJJo2y7GXSLvfGXVgfvvIE3wYTvzQws1U5+dk=", + "lastModified": 1669333226, + "narHash": "sha256-7yik4uYRAsCngs8AeZeYESCtU3LfCjjEaIQj7Sbjwhk=", "owner": "melange-re", "repo": "melange-compiler-libs", - "rev": "911e17f814430b5ea4c9e065884a53c323ae52bd", + "rev": "426463a77d0b70ecf0108c98e6a86d325cd01472", "type": "github" }, "original": { @@ -428,11 +448,11 @@ "nixpkgs": "nixpkgs_3" }, "locked": { - "lastModified": 1669153672, - "narHash": "sha256-tixGuv26m/gFKuFPnlVFJWkOknTlgZV5lYSpOJxTpZY=", + "lastModified": 1669332884, + "narHash": "sha256-Ok7aPVlT4Kt78YXM0QMkKzRG5XAc44p6FP+XAMIg1Cg=", "owner": "anmonteiro", "repo": "nix-overlays", - "rev": "48403bf49f57bccf942e421f499e37bc1109231c", + "rev": "3a834a703abe8f386273927e6119b400a538517b", "type": "github" }, "original": { @@ -441,6 +461,22 @@ "type": "github" } }, + "nix-pypi-fetcher": { + "flake": false, + "locked": { + "lastModified": 1669065297, + "narHash": "sha256-UStjXjNIuIm7SzMOWvuYWIHBkPUKQ8Id63BMJjnIDoA=", + "owner": "DavHau", + "repo": "nix-pypi-fetcher", + "rev": "a9885ac6a091576b5195d547ac743d45a2a615ac", + "type": "github" + }, + "original": { + "owner": "DavHau", + "repo": "nix-pypi-fetcher", + "type": "github" + } + }, "nixpkgs": { "inputs": { "flake-utils": [ @@ -450,11 +486,11 @@ "nixpkgs": "nixpkgs_2" }, "locked": { - "lastModified": 1668874951, - "narHash": "sha256-M4wgAmTc+ea0ltJ5EIWeAPzlh3SDXSE/Nfbpl4B/GEk=", + "lastModified": 1669332884, + "narHash": "sha256-Ok7aPVlT4Kt78YXM0QMkKzRG5XAc44p6FP+XAMIg1Cg=", "owner": "anmonteiro", "repo": "nix-overlays", - "rev": "35e06a888597e9cd955ea62a0afa1d21123fb24e", + "rev": "3a834a703abe8f386273927e6119b400a538517b", "type": "github" }, "original": { @@ -463,45 +499,63 @@ "type": "github" } }, + "nixpkgs-lib": { + "locked": { + "dir": "lib", + "lastModified": 1665349835, + "narHash": "sha256-UK4urM3iN80UXQ7EaOappDzcisYIuEURFRoGQ/yPkug=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "34c5293a71ffdb2fe054eb5288adc1882c1eb0b1", + "type": "github" + }, + "original": { + "dir": "lib", + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, "nixpkgs_2": { "locked": { - "lastModified": 1668829732, - "narHash": "sha256-T8mzu6/rN1i7kSQIOuxCPYy0BVRxfFy0CsSuXYVq+V8=", + "lastModified": 1669154743, + "narHash": "sha256-oqWVq/eYsjvIakytYlw3NBt0lcTQyXb7xDiJMDWIIf4=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "c4c1866143ec02016d30eabed22cd6d0de39a10b", + "rev": "977d4985a05cc68c9f3a208415aa2ef028b52772", "type": "github" }, "original": { "owner": "NixOS", "repo": "nixpkgs", - "rev": "c4c1866143ec02016d30eabed22cd6d0de39a10b", + "rev": "977d4985a05cc68c9f3a208415aa2ef028b52772", "type": "github" } }, "nixpkgs_3": { "locked": { - "lastModified": 1669067947, - "narHash": "sha256-DbTlGzUSj2j+3tMJ7UwV5ExGSg1WeZ+XZq6eAGpyd40=", + "lastModified": 1669154743, + "narHash": "sha256-oqWVq/eYsjvIakytYlw3NBt0lcTQyXb7xDiJMDWIIf4=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "802115d0227228b66b691f3b85b30b868976e8eb", + "rev": "977d4985a05cc68c9f3a208415aa2ef028b52772", "type": "github" }, "original": { "owner": "NixOS", "repo": "nixpkgs", - "rev": "802115d0227228b66b691f3b85b30b868976e8eb", + "rev": "977d4985a05cc68c9f3a208415aa2ef028b52772", "type": "github" } }, "nixpkgs_4": { "locked": { - "lastModified": 1669076005, - "narHash": "sha256-uzMji2q9Pk3jUH+e5nEFtoOZCP4VV1PDRJRLVmriY0M=", + "lastModified": 1669261230, + "narHash": "sha256-AjddxRPd5y5jge77281P3O8+Cnafj842Xg59rwV4x+0=", "owner": "nixos", "repo": "nixpkgs", - "rev": "69335c46c48a73f291d5c6f332fb9fe8b8e22b30", + "rev": "8e8b5f3b1e899bf5d250279578c0283705b8cdb4", "type": "github" }, "original": { @@ -567,11 +621,11 @@ "opam-repository": "opam-repository" }, "locked": { - "lastModified": 1669129897, - "narHash": "sha256-PJN0SBJHShc+glORz8X0/jTSdMqg2/qhDYuDlrhdXto=", + "lastModified": 1669304352, + "narHash": "sha256-SZJUZ9H3xeJzkiEUxbgzuvwWhZFLmzDbQhF24jNte+I=", "ref": "refs/heads/master", - "rev": "7bffb3a5eabd2584573476a24a909672bb5d4113", - "revCount": 1878, + "rev": "d079baa1a088f924ee92c9b0b68801488cccfc18", + "revCount": 1881, "submodules": true, "type": "git", "url": "https://www.github.com/ocaml/ocaml-lsp" @@ -622,11 +676,11 @@ "opam2json": "opam2json_2" }, "locked": { - "lastModified": 1667491541, - "narHash": "sha256-Yq/wEQC7fz4A5zIhu6fymheUJdWBKx/Te7PMMb7lS2w=", + "lastModified": 1669188829, + "narHash": "sha256-0nQkXV3QkTrcSJgQH0OHVThyDmprxuTaHc8eb7SHr04=", "owner": "tweag", "repo": "opam-nix", - "rev": "c3db5fdf0f2b4aba3f052a9b9b787303aa268a2f", + "rev": "c26fd47558562c5345fc123aa11989e7e6b28b0f", "type": "github" }, "original": { @@ -686,11 +740,11 @@ "opam-repository_2": { "flake": false, "locked": { - "lastModified": 1669135050, - "narHash": "sha256-a3kRBkuRXidmxEsk6u38VmKKlPwoNaM4hgY8u+kRFdw=", + "lastModified": 1669320500, + "narHash": "sha256-lP45g0eOVftNYIbGf2tIp2OoOfHk113XNs8KlQ+gx00=", "owner": "ocaml", "repo": "opam-repository", - "rev": "ab8d6d9033637382c11429e5b155148ad49127a5", + "rev": "e5921bc6ce2143c3c9b9b0c54ce423ab2e947433", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index ad68aa2f95a..4209e95533d 100644 --- a/flake.nix +++ b/flake.nix @@ -96,7 +96,13 @@ }; devShells.slim = - let pkgs = nix-overlays.legacyPackages.${system}; + let + pkgs = nix-overlays.legacyPackages.${system}.appendOverlays [ + (self: super: { + ocamlPackages = self.ocaml-ng.ocamlPackages_4_14; + }) + melange.overlays.default + ]; in pkgs.mkShell { nativeBuildInputs = with pkgs; [ pkg-config nodejs-slim ]; diff --git a/src/dune_rules/module_compilation.ml b/src/dune_rules/module_compilation.ml index 8dc037cc626..5467eea4643 100644 --- a/src/dune_rules/module_compilation.ml +++ b/src/dune_rules/module_compilation.ml @@ -8,11 +8,7 @@ module CC = Compilation_context extension is not .ml or when the .ml and .mli are in different directories. This flags makes the compiler think there is a .mli file and will the read the cmi file rather than create it. *) -let force_read_cmi ~(cm_kind : Lib_mode.Cm_kind.t) source_file = - let args = [ "-intf-suffix"; Path.extension source_file ] in - match cm_kind with - | Melange Cmj -> "--bs-read-cmi" :: args - | Ocaml (Cmo | Cmx | Cmi) | Melange Cmi -> args +let force_read_cmi source_file = [ "-intf-suffix"; Path.extension source_file ] (* Build the cm* if the corresponding source is present, in the case of cmi if the mli is not present it is added as additional target to the .cmo @@ -116,7 +112,7 @@ let build_cm cctx ~force_write_cmi ~precompiled_cmi ~cm_kind (m : Module.t) in let open Memo.O in let* extra_args, extra_deps, other_targets = - if precompiled_cmi then Memo.return (force_read_cmi ~cm_kind src, [], []) + if precompiled_cmi then Memo.return (force_read_cmi src, [], []) else (* If we're compiling an implementation, then the cmi is present *) let public_vlib_module = Module.kind m = Impl_vmodule in @@ -138,7 +134,7 @@ let build_cm cctx ~force_write_cmi ~precompiled_cmi ~cm_kind (m : Module.t) | (Ocaml (Cmo | Cmx) | Melange Cmj), _, _ -> let cmi_kind = Lib_mode.Cm_kind.cmi cm_kind in Memo.return - ( force_read_cmi ~cm_kind src + ( force_read_cmi src , [ Path.build (Obj_dir.Module.cm_file_exn obj_dir m ~kind:cmi_kind) ] , [] ) From 7993a448b4ac30ffd0329067d19af33ac9016c21 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Fri, 25 Nov 2022 00:01:50 -0500 Subject: [PATCH 02/20] test(vlib): document implementation aliasing scheme (#6574) Signed-off-by: Rudi Grinberg --- .../impl-private-modules.t/run.t | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/test/blackbox-tests/test-cases/virtual-libraries/impl-private-modules.t/run.t b/test/blackbox-tests/test-cases/virtual-libraries/impl-private-modules.t/run.t index 509a4e409b6..861ca1cf1ac 100644 --- a/test/blackbox-tests/test-cases/virtual-libraries/impl-private-modules.t/run.t +++ b/test/blackbox-tests/test-cases/virtual-libraries/impl-private-modules.t/run.t @@ -2,3 +2,24 @@ They can only introduce private modules: $ dune build --debug-dependency-path Private module Baz implementing bar + +Note the aliasing scheme for implementations differs form other libraries. In +particular, we use a longer prefix for private modules of implementations so +that they never collide with modules present in the virtual library. + + $ cat _build/default/impl/foo__foo_impl__.ml-gen + (* generated by dune *) + + (** @canonical Foo.Bar *) + module Bar = Foo__Bar + + (** @canonical Foo.Priv *) + module Priv = Foo__foo_impl____Priv + +Here we look at the raw artifacts for our implementation and verify it matches +the alias: + + $ ls _build/default/impl/.foo_impl.objs/byte/*.cmi + _build/default/impl/.foo_impl.objs/byte/foo__Bar.cmi + _build/default/impl/.foo_impl.objs/byte/foo__foo_impl__.cmi + _build/default/impl/.foo_impl.objs/byte/foo__foo_impl____Priv.cmi From dd6f9763cde3ecb30f2af4adc59f92eedb0de8ae Mon Sep 17 00:00:00 2001 From: Ali Caglayan Date: Fri, 25 Nov 2022 02:06:43 +0100 Subject: [PATCH 03/20] test(coq): add extraction example with patch Signed-off-by: Ali Caglayan --- .../test-cases/coq/extraction-patch.t/dune | 14 +++++++++++++ .../coq/extraction-patch.t/dune-project | 2 ++ .../coq/extraction-patch.t/extraction/dune | 4 ++++ .../extraction/my_coq_file.v | 8 +++++++ .../coq/extraction-patch.t/my_prog.ml | 10 +++++++++ .../test-cases/coq/extraction-patch.t/run.t | 21 +++++++++++++++++++ 6 files changed, 59 insertions(+) create mode 100644 test/blackbox-tests/test-cases/coq/extraction-patch.t/dune create mode 100644 test/blackbox-tests/test-cases/coq/extraction-patch.t/dune-project create mode 100644 test/blackbox-tests/test-cases/coq/extraction-patch.t/extraction/dune create mode 100644 test/blackbox-tests/test-cases/coq/extraction-patch.t/extraction/my_coq_file.v create mode 100644 test/blackbox-tests/test-cases/coq/extraction-patch.t/my_prog.ml create mode 100644 test/blackbox-tests/test-cases/coq/extraction-patch.t/run.t diff --git a/test/blackbox-tests/test-cases/coq/extraction-patch.t/dune b/test/blackbox-tests/test-cases/coq/extraction-patch.t/dune new file mode 100644 index 00000000000..bb6a9e520db --- /dev/null +++ b/test/blackbox-tests/test-cases/coq/extraction-patch.t/dune @@ -0,0 +1,14 @@ +(executable + (name my_prog)) + +(rule + (target my_coq_file.ml) + (deps extraction/my_coq_file.ml) + (action + (with-stdout-to %{target} + ; here you patch stuff + (cat %{deps})))) + +(copy_files extraction/Datatypes.ml) +(copy_files extraction/Datatypes.mli) +(copy_files extraction/my_coq_file.mli) diff --git a/test/blackbox-tests/test-cases/coq/extraction-patch.t/dune-project b/test/blackbox-tests/test-cases/coq/extraction-patch.t/dune-project new file mode 100644 index 00000000000..98d5258f3e8 --- /dev/null +++ b/test/blackbox-tests/test-cases/coq/extraction-patch.t/dune-project @@ -0,0 +1,2 @@ +(lang dune 3.7) +(using coq 0.7) diff --git a/test/blackbox-tests/test-cases/coq/extraction-patch.t/extraction/dune b/test/blackbox-tests/test-cases/coq/extraction-patch.t/extraction/dune new file mode 100644 index 00000000000..97a43d4ac16 --- /dev/null +++ b/test/blackbox-tests/test-cases/coq/extraction-patch.t/extraction/dune @@ -0,0 +1,4 @@ +(coq.extraction + (mode vo) + (prelude my_coq_file) + (extracted_modules Datatypes my_coq_file)) diff --git a/test/blackbox-tests/test-cases/coq/extraction-patch.t/extraction/my_coq_file.v b/test/blackbox-tests/test-cases/coq/extraction-patch.t/extraction/my_coq_file.v new file mode 100644 index 00000000000..9a6003556c6 --- /dev/null +++ b/test/blackbox-tests/test-cases/coq/extraction-patch.t/extraction/my_coq_file.v @@ -0,0 +1,8 @@ +Definition nb (b : bool) : bool := + match b with + | false => true + | true => false + end. + +Require Extraction. +Separate Extraction nb. diff --git a/test/blackbox-tests/test-cases/coq/extraction-patch.t/my_prog.ml b/test/blackbox-tests/test-cases/coq/extraction-patch.t/my_prog.ml new file mode 100644 index 00000000000..137c750150a --- /dev/null +++ b/test/blackbox-tests/test-cases/coq/extraction-patch.t/my_prog.ml @@ -0,0 +1,10 @@ +let bool_to_string (b : Datatypes.bool) : string = + match b with + | Datatypes.Coq_true -> "true" + | Datatypes.Coq_false -> "false" + +let () = +Datatypes.Coq_true +|> My_coq_file.nb +|> bool_to_string +|> Printf.printf "Result: %s\n" \ No newline at end of file diff --git a/test/blackbox-tests/test-cases/coq/extraction-patch.t/run.t b/test/blackbox-tests/test-cases/coq/extraction-patch.t/run.t new file mode 100644 index 00000000000..c8259f94712 --- /dev/null +++ b/test/blackbox-tests/test-cases/coq/extraction-patch.t/run.t @@ -0,0 +1,21 @@ + + $ dune build + + $ ls _build/default + Datatypes.ml + Datatypes.mli + extraction + my_coq_file.ml + my_coq_file.mli + my_prog.exe + my_prog.ml + my_prog.mli + + $ cat _build/default/Datatypes.ml + + type bool = + | Coq_true + | Coq_false + + $ dune exec -- ./my_prog.exe + Result: false From a29a215f4340649812ee852b04faacb90b4efadb Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Fri, 25 Nov 2022 11:01:11 -0500 Subject: [PATCH 04/20] fix: format alternative file names (#6566) alternative file names such as dune-file should be accepted Signed-off-by: Rudi Grinberg --- CHANGES.md | 3 + src/dune_rules/format_rules.ml | 90 +++++++++++-------- .../formatting/format-alternative.t | 9 +- 3 files changed, 62 insertions(+), 40 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 0730da53162..ac1c32fc3c2 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,9 @@ Unreleased ---------- +- Format dune files when they are named `dune-file`. This occurs when we enable + the alternative file names project option. (#6566, @rgrinberg) + - Do not shadow library interface modules (#6549, fixes #6545, @rgrinberg) - Move `$ dune ocaml-merlin -dump-config=$dir` to `$ dune ocaml merlin diff --git a/src/dune_rules/format_rules.ml b/src/dune_rules/format_rules.ml index acb51757576..80f084ab008 100644 --- a/src/dune_rules/format_rules.ml +++ b/src/dune_rules/format_rules.ml @@ -65,60 +65,72 @@ let gen_rules_output sctx (config : Format_config.t) ~version ~dialects let source_dir = Path.Build.drop_build_context_exn dir in let alias_formatted = Alias.fmt ~dir:output_dir in let depend_on_files named = depend_on_files ~named (Path.build dir) in + let open Memo.O in let setup_formatting file = let input_basename = Path.Source.basename file in let input = Path.Build.relative dir input_basename in let output = Path.Build.relative output_dir input_basename in let formatter = - let input = Path.build input in - match Path.Source.basename file with - | "dune" when Format_config.includes config Dune -> - Option.some - @@ Action_builder.with_file_targets ~file_targets:[ output ] - @@ - let open Action_builder.O in - let+ () = Action_builder.path input in - Action.Full.make (action ~version input output) - | _ -> - let ext = Path.Source.extension file in - let open Option.O in - let* dialect, kind = Dialect.DB.find_by_extension dialects ext in - let* () = - Option.some_if - (Format_config.includes config (Dialect (Dialect.name dialect))) - () - in - let+ loc, action, extra_deps = - match Dialect.format dialect kind with - | Some _ as action -> action - | None -> ( - match Dialect.preprocess dialect kind with - | None -> Dialect.format Dialect.ocaml kind - | Some _ -> None) - in - let src = Path.as_in_build_dir_exn input in - let extra_deps = - match extra_deps with - | [] -> Action_builder.return () - | extra_deps -> depend_on_files extra_deps - in - let open Action_builder.With_targets.O in - Action_builder.with_no_targets extra_deps - >>> Preprocessing.action_for_pp_with_target - ~sandbox:Sandbox_config.default ~loc ~expander ~action ~src - ~target:output + let ext = Path.Source.extension file in + let open Option.O in + let* dialect, kind = Dialect.DB.find_by_extension dialects ext in + let* () = + Option.some_if + (Format_config.includes config (Dialect (Dialect.name dialect))) + () + in + let+ loc, action, extra_deps = + match Dialect.format dialect kind with + | Some _ as action -> action + | None -> ( + match Dialect.preprocess dialect kind with + | None -> Dialect.format Dialect.ocaml kind + | Some _ -> None) + in + let extra_deps = + match extra_deps with + | [] -> Action_builder.return () + | extra_deps -> depend_on_files extra_deps + in + let open Action_builder.With_targets.O in + Action_builder.with_no_targets extra_deps + >>> Preprocessing.action_for_pp_with_target + ~sandbox:Sandbox_config.default ~loc ~expander ~action ~src:input + ~target:output in Memo.Option.iter formatter ~f:(fun action -> - let open Memo.O in Super_context.add_rule sctx ~mode:Standard ~loc ~dir action >>> add_diff sctx loc alias_formatted ~dir ~input:(Path.build input) ~output) in - let open Memo.O in let* () = Source_tree.files_of source_dir >>= Memo.parallel_iter_set (module Path.Source.Set) ~f:setup_formatting in + let* () = + match Format_config.includes config Dune with + | false -> Memo.return () + | true -> ( + Source_tree.find_dir source_dir >>= function + | None -> Memo.return () + | Some source_dir -> ( + match Source_tree.Dir.dune_file source_dir with + | None -> Memo.return () + | Some f -> + let path = Source_tree.Dune_file.path f in + let input_basename = Path.Source.basename path in + let input = Path.Build.relative dir input_basename in + let output = Path.Build.relative output_dir input_basename in + Super_context.add_rule sctx ~mode:Standard ~loc ~dir + (Action_builder.with_file_targets ~file_targets:[ output ] + @@ + let open Action_builder.O in + let input = Path.build input in + let+ () = Action_builder.path input in + Action.Full.make (action ~version input output)) + >>> add_diff sctx loc alias_formatted ~dir ~input:(Path.build input) + ~output)) + in Rules.Produce.Alias.add_deps alias_formatted (Action_builder.return ()) let gen_rules sctx ~output_dir = diff --git a/test/blackbox-tests/test-cases/formatting/format-alternative.t b/test/blackbox-tests/test-cases/formatting/format-alternative.t index 8471648ca2b..af427a0c063 100644 --- a/test/blackbox-tests/test-cases/formatting/format-alternative.t +++ b/test/blackbox-tests/test-cases/formatting/format-alternative.t @@ -11,7 +11,14 @@ Dune files names dune-file should be formatted > EOF $ dune fmt + File "dune-file", line 1, characters 0-0: + Error: Files _build/default/dune-file and _build/default/.formatted/dune-file + differ. + Promoting _build/default/.formatted/dune-file to dune-file. + [1] $ cat dune-file (rule - (with-stdout-to foo (echo bar))) + (with-stdout-to + foo + (echo bar))) From 1f66b23f3b04c1bd99d161b0403c9f6200bdf661 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Fri, 25 Nov 2022 11:01:35 -0500 Subject: [PATCH 05/20] refactor: rename toplevel dune to dune-file (#6565) Signed-off-by: Rudi Grinberg --- dune => dune-file | 0 dune-project | 2 ++ 2 files changed, 2 insertions(+) rename dune => dune-file (100%) diff --git a/dune b/dune-file similarity index 100% rename from dune rename to dune-file diff --git a/dune-project b/dune-project index 06c24e3295e..426419d020a 100644 --- a/dune-project +++ b/dune-project @@ -6,6 +6,8 @@ (generate_opam_files true) +(accept_alternative_dune_file_name) + (strict_package_deps false) ; Reserved for Dune itself. This is to help with the bootstrap From 3b7e2ca3d3058d4189ecd17f995d481a3a77860a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Ch=C3=A1varri?= Date: Fri, 25 Nov 2022 17:05:28 +0100 Subject: [PATCH 06/20] fix(melange): handle missing melc We now generate the melange rules even if melc is absent. This gives us a good error message whenever melc is missing. Fixes #6403 Signed-off-by: Javier Chavarri Co-authored-by: Rudi Grinberg --- src/dune_rules/module_compilation.ml | 19 ++-- .../test-cases/melange/missing-melc.t | 97 +++++++++++++++++++ 2 files changed, 109 insertions(+), 7 deletions(-) create mode 100644 test/blackbox-tests/test-cases/melange/missing-melc.t diff --git a/src/dune_rules/module_compilation.ml b/src/dune_rules/module_compilation.ml index 5467eea4643..ee9b49c6de2 100644 --- a/src/dune_rules/module_compilation.ml +++ b/src/dune_rules/module_compilation.ml @@ -94,12 +94,17 @@ let build_cm cctx ~force_write_cmi ~precompiled_cmi ~cm_kind (m : Module.t) in let open Memo.O in let* compiler = - let+ compiler = - match mode with - | Ocaml mode -> Memo.return @@ Context.compiler ctx mode - | Melange -> Melange_binary.melc sctx ~dir - in - Result.to_option compiler + match mode with + | Melange -> + let+ melc = Melange_binary.melc sctx ~dir in + Some melc + | Ocaml mode -> + Memo.return + (let compiler = Context.compiler ctx mode in + (* TODO one day remove this silly optimization *) + match compiler with + | Ok _ as s -> Some s + | Error _ -> None) in (let open Option.O in let* compiler = compiler in @@ -239,7 +244,7 @@ let build_cm cctx ~force_write_cmi ~precompiled_cmi ~cm_kind (m : Module.t) (let open Action_builder.With_targets.O in Action_builder.with_no_targets (Action_builder.paths extra_deps) >>> Action_builder.with_no_targets other_cm_files - >>> Command.run ~dir:(Path.build ctx.build_dir) (Ok compiler) + >>> Command.run ~dir:(Path.build ctx.build_dir) compiler [ Command.Args.dyn flags ; cmt_args ; Command.Args.S obj_dirs diff --git a/test/blackbox-tests/test-cases/melange/missing-melc.t b/test/blackbox-tests/test-cases/melange/missing-melc.t new file mode 100644 index 00000000000..cad9c9e7534 --- /dev/null +++ b/test/blackbox-tests/test-cases/melange/missing-melc.t @@ -0,0 +1,97 @@ +Test cases when melc is not available + + $ cat > dune-project < (lang dune 3.6) + > (using melange 0.1) + > EOF + + $ cat > main_melange.ml < let () = + > print_endline "hello from melange" + > EOF + +Set up some fake environment without melc + + $ mkdir _path + $ ln -s $(command -v dune) _path/ + $ ln -s $(command -v ocamlc) _path/ + $ ln -s $(command -v ocamldep) _path/ + +For melange.emit stanzas, an error is shown + + $ cat > dune < (melange.emit + > (target output) + > (entries main_melange) + > (module_system commonjs)) + > EOF + + $ (unset INSIDE_DUNE; PATH=_path dune build --always-show-command-line --root . output/main_melange.js) + File "dune", line 1, characters 0-81: + 1 | (melange.emit + 2 | (target output) + 3 | (entries main_melange) + 4 | (module_system commonjs)) + Error: Program melc not found in the tree or in PATH + (context: default) + Hint: opam install melange + [1] + +For libraries, if no melange.emit stanza is found, build does not fail + + $ cat > dune < (library + > (name lib1) + > (modules :standard \ main_native) + > (modes byte melange)) + > (executable + > (name main_native) + > (modules main_native) + > (modes exe byte) + > (libraries lib1)) + > EOF + + $ cat > main_native.ml < let () = + > print_endline Lib1.Lib.t + > EOF + + $ cat > lib.ml < let t = "hello from native" + > EOF + + $ (unset INSIDE_DUNE; PATH=_path dune build --always-show-command-line --root . main_native.bc) + $ dune exec ./main_native.bc + hello from native + +If melange.emit stanza is found, but no rules are executed, build does not fail + + $ cat > dune < (library + > (name lib1) + > (modules :standard \ main_native main_melange) + > (modes byte melange)) + > (executable + > (name main_native) + > (modules main_native) + > (modes exe byte) + > (libraries lib1)) + > (melange.emit + > (target output) + > (entries main_melange) + > (libraries lib1) + > (module_system commonjs)) + > EOF + + $ (unset INSIDE_DUNE; PATH=_path dune build --always-show-command-line --root . main_native.bc) + $ dune exec ./main_native.bc + hello from native + +But trying to build any melange artifacts will fail + + $ (unset INSIDE_DUNE; PATH=_path dune build --always-show-command-line --root . output/main_melange.js) + File ".lib1.objs/melange/_unknown_", line 1, characters 0-0: + Error: Program melc not found in the tree or in PATH + (context: default) + Hint: opam install melange + [1] From 4d6825b80d971c7d045c4d2a72a91a24d4dc24b2 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Sat, 26 Nov 2022 02:05:31 -0500 Subject: [PATCH 07/20] test: document test gh4907 (#6586) Signed-off-by: Rudi Grinberg --- test/blackbox-tests/test-cases/subdir-stanza/gh4907.t | 1 + 1 file changed, 1 insertion(+) diff --git a/test/blackbox-tests/test-cases/subdir-stanza/gh4907.t b/test/blackbox-tests/test-cases/subdir-stanza/gh4907.t index 8b3618a882c..e61ece7d8bc 100644 --- a/test/blackbox-tests/test-cases/subdir-stanza/gh4907.t +++ b/test/blackbox-tests/test-cases/subdir-stanza/gh4907.t @@ -1,3 +1,4 @@ +Directories with spaces $ cat <dune-project > (lang dune 3.0) From 4b66e9c4741a22fb01b67777b0d70465dfd2ecb2 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Sat, 26 Nov 2022 02:05:59 -0500 Subject: [PATCH 08/20] test: add pkg-config dependency (#6582) Signed-off-by: Rudi Grinberg --- otherlibs/configurator/test/blackbox-tests/dune | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/otherlibs/configurator/test/blackbox-tests/dune b/otherlibs/configurator/test/blackbox-tests/dune index 3049871e875..528c0906f52 100644 --- a/otherlibs/configurator/test/blackbox-tests/dune +++ b/otherlibs/configurator/test/blackbox-tests/dune @@ -3,6 +3,10 @@ (package dune) (package dune-configurator))) +(cram + (applies_to pkg-config-quoting) + (deps %{bin:pkg-config})) + (cram (enabled_if (<> %{ocaml-config:system} win)) From a790dd36ae51ab13676e58cfff60b08a15508d93 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Sat, 26 Nov 2022 02:15:49 -0500 Subject: [PATCH 09/20] fix: dune files from source tree (#6585) The dune file path returned by Source_tree.Dune_file is always present. This is wrong because some dune files are generated using subdir. This commit tweaks the API to reflect this fact and updates the rest of the code to handle such virtual dune files. Signed-off-by: Rudi Grinberg --- src/dune_engine/source_tree.ml | 73 +++++++++++++++++++++------------ src/dune_engine/source_tree.mli | 4 +- src/dune_engine/sub_dirs.ml | 14 +++++++ src/dune_engine/sub_dirs.mli | 6 +++ src/dune_rules/dir_contents.ml | 9 +++- src/dune_rules/dune_file.ml | 14 +++++-- src/dune_rules/dune_file.mli | 2 +- src/dune_rules/dune_load.ml | 13 +++++- src/dune_rules/format_rules.ml | 30 +++++++------- 9 files changed, 115 insertions(+), 50 deletions(-) diff --git a/src/dune_engine/source_tree.ml b/src/dune_engine/source_tree.ml index 024bf51a0df..8578d09d86b 100644 --- a/src/dune_engine/source_tree.ml +++ b/src/dune_engine/source_tree.ml @@ -36,6 +36,13 @@ module Dune_file = struct { contents : Sub_dirs.Dir_map.per_dir ; for_subdirs : Sub_dirs.Dir_map.t } + + let to_dyn { contents; for_subdirs } = + let open Dyn in + record + [ ("contents", Sub_dirs.Dir_map.dyn_of_per_dir contents) + ; ("for_subdirs", Sub_dirs.Dir_map.to_dyn for_subdirs) + ] end let fname = "dune" @@ -46,13 +53,25 @@ module Dune_file = struct | Plain | Ocaml_script + let dyn_of_kind = function + | Plain -> Dyn.variant "Plain" [] + | Ocaml_script -> Dyn.variant "Ocaml_script" [] + type t = - { path : Path.Source.t + { path : Path.Source.t option ; kind : kind ; (* for [kind = Ocaml_script], this is the part inserted with subdir *) plain : Plain.t } + let to_dyn { path; kind; plain } = + let open Dyn in + record + [ ("path", option Path.Source.to_dyn path) + ; ("kind", dyn_of_kind kind) + ; ("plain", Plain.to_dyn plain) + ] + let get_static_sexp t = t.plain.contents.sexps let kind t = t.kind @@ -67,15 +86,18 @@ module Dune_file = struct let load_plain sexps ~file ~from_parent ~project = let+ active = let+ parsed = - let decoder = - { Sub_dirs.decode = - (fun ast d -> - let d = Dune_project.set_parsing_context project d in - Dune_lang.Decoder.parse d Univ_map.empty - (Dune_lang.Ast.List (Loc.none, ast))) - } - in - Sub_dirs.decode ~file decoder sexps + match file with + | None -> Memo.return Sub_dirs.Dir_map.empty + | Some file -> + let decoder = + { Sub_dirs.decode = + (fun ast d -> + let d = Dune_project.set_parsing_context project d in + Dune_lang.Decoder.parse d Univ_map.empty + (Dune_lang.Ast.List (Loc.none, ast))) + } + in + Sub_dirs.decode ~file decoder sexps in match from_parent with | None -> parsed @@ -84,14 +106,14 @@ module Dune_file = struct let contents = Sub_dirs.Dir_map.root active in { Plain.contents; for_subdirs = active } - let load file ~file_exists ~from_parent ~project = + let load file ~from_parent ~project = let+ kind, plain = let load_plain = load_plain ~file ~from_parent ~project in - match file_exists with - | false -> + match file with + | None -> let+ plain = load_plain [] in (Plain, plain) - | true -> + | Some file -> let* kind, ast = Fs_memo.with_lexbuf_from_file (In_source_dir file) ~f:(fun lb -> let kind, ast = @@ -511,7 +533,7 @@ end = struct | Error -> Memo.exec memo (`Is_error true, inp) let dune_file ~(dir_status : Sub_dirs.Status.t) ~path ~files ~project = - let file_exists = + let file = if dir_status = Data_only then None else if Dune_project.accept_alternative_dune_file_name project @@ -534,18 +556,15 @@ end = struct in (dune_file.path, dir_map) in - let file = - match (file_exists, from_parent) with - | None, None -> None - | Some fname, _ -> Some (Path.Source.relative path fname) - | None, Some (path, _) -> Some path - in - Memo.Option.map file ~f:(fun file -> - let open Memo.O in - let* () = ensure_dune_project_file_exists project in - let file_exists = Option.is_some file_exists in - let from_parent = Option.map from_parent ~f:snd in - Dune_file.load file ~file_exists ~project ~from_parent) + let open Memo.O in + match (from_parent, file) with + | None, None -> Memo.return None + | _, _ -> + let* () = ensure_dune_project_file_exists project in + let file = Option.map file ~f:(Path.Source.relative path) in + let from_parent = Option.map from_parent ~f:snd in + let+ dune_file = Dune_file.load file ~project ~from_parent in + Some dune_file let contents { Readdir.path; dirs; files } ~dirs_visited ~project ~(dir_status : Sub_dirs.Status.t) = diff --git a/src/dune_engine/source_tree.mli b/src/dune_engine/source_tree.mli index 3433eb116c9..a194b790086 100644 --- a/src/dune_engine/source_tree.mli +++ b/src/dune_engine/source_tree.mli @@ -13,11 +13,13 @@ module Dune_file : sig type t + val to_dyn : t -> Dyn.t + val get_static_sexp : t -> Dune_lang.Ast.t list val kind : t -> kind - val path : t -> Path.Source.t + val path : t -> Path.Source.t option end module Dir : sig diff --git a/src/dune_engine/sub_dirs.ml b/src/dune_engine/sub_dirs.ml index f5805183358..4eb4e085226 100644 --- a/src/dune_engine/sub_dirs.ml +++ b/src/dune_engine/sub_dirs.ml @@ -155,11 +155,25 @@ module Dir_map = struct ; subdir_status : subdir_stanzas } + let dyn_of_per_dir { sexps; subdir_status = _ } = + let open Dyn in + record + [ ( "sexps" + , list Dune_lang.to_dyn (List.map ~f:Dune_lang.Ast.remove_locs sexps) ) + ] + type t = { data : per_dir ; nodes : t String.Map.t } + let rec to_dyn { data; nodes } = + let open Dyn in + record + [ ("data", dyn_of_per_dir data) + ; ("nodes", String.Map.to_dyn to_dyn nodes) + ] + let empty_per_dir = { sexps = []; subdir_status = Status.Map.init ~f:(fun _ -> None) } diff --git a/src/dune_engine/sub_dirs.mli b/src/dune_engine/sub_dirs.mli index 7ab80fb05c9..a36274e13ce 100644 --- a/src/dune_engine/sub_dirs.mli +++ b/src/dune_engine/sub_dirs.mli @@ -59,6 +59,12 @@ module Dir_map : sig ; subdir_status : subdir_stanzas } + val dyn_of_per_dir : per_dir -> Dyn.t + + val to_dyn : t -> Dyn.t + + val empty : t + val descend : t -> string -> t option val sub_dirs : t -> string list diff --git a/src/dune_rules/dir_contents.ml b/src/dune_rules/dir_contents.ml index c612aecd3c5..50296bdcf13 100644 --- a/src/dune_rules/dir_contents.ml +++ b/src/dune_rules/dir_contents.ml @@ -9,8 +9,13 @@ open Memo.O let loc_of_dune_file st_dir = Loc.in_file (Path.source - (match Source_tree.Dir.dune_file st_dir with - | Some d -> Source_tree.Dune_file.path d + (match + let open Option.O in + let* dune_file = Source_tree.Dir.dune_file st_dir in + (* TODO not really correct. we need to know the [(subdir ..)] that introduced this *) + Source_tree.Dune_file.path dune_file + with + | Some s -> s | None -> Path.Source.relative (Source_tree.Dir.path st_dir) "_unknown_")) type t = diff --git a/src/dune_rules/dune_file.ml b/src/dune_rules/dune_file.ml index 77ac2593cb2..bc838e3fdba 100644 --- a/src/dune_rules/dune_file.ml +++ b/src/dune_rules/dune_file.ml @@ -2422,11 +2422,19 @@ module Stanzas = struct parse_file_includes ~stanza_parser ~context sexps | stanza -> Memo.return [ stanza ]) - let parse ~file (project : Dune_project.t) sexps = + let parse ~file ~dir (project : Dune_project.t) sexps = let stanza_parser = parser project in let open Memo.O in let+ stanzas = - let context = Include_stanza.in_file file in + let context = + Include_stanza.in_file + @@ + match file with + | Some f -> f + | None -> + (* TODO this is wrong *) + Path.Source.relative dir Source_tree.Dune_file.fname + in parse_file_includes ~stanza_parser ~context sexps in let (_ : bool) = @@ -2476,7 +2484,7 @@ let is_promoted_rule version rule = let parse sexps ~dir ~file ~project = let open Memo.O in - let+ stanzas = Stanzas.parse ~file project sexps in + let+ stanzas = Stanzas.parse ~file ~dir project sexps in let stanzas = if !Clflags.ignore_promoted_rules then let version = Dune_project.dune_version project in diff --git a/src/dune_rules/dune_file.mli b/src/dune_rules/dune_file.mli index 0833bb8b21d..b8b63958a94 100644 --- a/src/dune_rules/dune_file.mli +++ b/src/dune_rules/dune_file.mli @@ -507,7 +507,7 @@ val to_dyn : t -> Dyn.t val parse : Dune_lang.Ast.t list -> dir:Path.Source.t - -> file:Path.Source.t + -> file:Path.Source.t option -> project:Dune_project.t -> t Memo.t diff --git a/src/dune_rules/dune_load.ml b/src/dune_rules/dune_load.ml index 8202050d764..c941f022b37 100644 --- a/src/dune_rules/dune_load.ml +++ b/src/dune_rules/dune_load.ml @@ -166,7 +166,7 @@ module Script = struct Path.build generated_dune_file |> Io.Untracked.with_lexbuf_from_file ~f:(Dune_lang.Parser.parse ~mode:Many) |> List.rev_append from_parent - |> Dune_file.parse ~dir ~file ~project + |> Dune_file.parse ~dir ~file:(Some file) ~project let eval_one = let module Input = struct @@ -196,7 +196,16 @@ module Dune_files = struct match Source_tree.Dune_file.kind dune_file with | Ocaml_script -> Memo.return - (Script { script = { dir; project; file }; from_parent = static }) + (Script + { script = + { dir + ; project + ; file = + (* we can't introduce ocaml syntax with [(sudir ..)] *) + Option.value_exn file + } + ; from_parent = static + }) | Plain -> let open Memo.O in let+ stanzas = Dune_file.parse static ~dir ~file ~project in diff --git a/src/dune_rules/format_rules.ml b/src/dune_rules/format_rules.ml index 80f084ab008..e8bcc67d31d 100644 --- a/src/dune_rules/format_rules.ml +++ b/src/dune_rules/format_rules.ml @@ -116,20 +116,22 @@ let gen_rules_output sctx (config : Format_config.t) ~version ~dialects | Some source_dir -> ( match Source_tree.Dir.dune_file source_dir with | None -> Memo.return () - | Some f -> - let path = Source_tree.Dune_file.path f in - let input_basename = Path.Source.basename path in - let input = Path.Build.relative dir input_basename in - let output = Path.Build.relative output_dir input_basename in - Super_context.add_rule sctx ~mode:Standard ~loc ~dir - (Action_builder.with_file_targets ~file_targets:[ output ] - @@ - let open Action_builder.O in - let input = Path.build input in - let+ () = Action_builder.path input in - Action.Full.make (action ~version input output)) - >>> add_diff sctx loc alias_formatted ~dir ~input:(Path.build input) - ~output)) + | Some f -> ( + match Source_tree.Dune_file.path f with + | None -> Memo.return () + | Some path -> + let input_basename = Path.Source.basename path in + let input = Path.Build.relative dir input_basename in + let output = Path.Build.relative output_dir input_basename in + Super_context.add_rule sctx ~mode:Standard ~loc ~dir + (Action_builder.with_file_targets ~file_targets:[ output ] + @@ + let open Action_builder.O in + let input = Path.build input in + let+ () = Action_builder.path input in + Action.Full.make (action ~version input output)) + >>> add_diff sctx loc alias_formatted ~dir ~input:(Path.build input) + ~output))) in Rules.Produce.Alias.add_deps alias_formatted (Action_builder.return ()) From 065db73cf9baab84aa4917eb20ea79a2fb33f735 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Sat, 26 Nov 2022 09:44:50 -0500 Subject: [PATCH 10/20] refactor: remove dead code (#6580) we already compute deps for [Wrapped_compat] in [Dep_rules] Signed-off-by: Rudi Grinberg --- src/dune_rules/modules.ml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/dune_rules/modules.ml b/src/dune_rules/modules.ml index a35a551d157..59a1e2137d1 100644 --- a/src/dune_rules/modules.ml +++ b/src/dune_rules/modules.ml @@ -355,10 +355,6 @@ module Wrapped = struct let find_dep t ~of_ name = match Module.kind of_ with | Alias -> None - | Wrapped_compat -> ( - match lib_interface t with - | Some li -> Option.some_if (name = Module.name li) li - | None -> Module_name.Map.find t.modules name) | _ -> if is_alias_name t name then Some t.alias_module else Module_name.Map.find t.modules name From a7eb2544575af7cf591de5ddc036a9d9dfb0cedf Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Sat, 26 Nov 2022 10:22:07 -0500 Subject: [PATCH 11/20] refactor: define the .bin magic path once (#6578) Signed-off-by: Rudi Grinberg --- src/dune_rules/artifacts.ml | 4 +++- src/dune_rules/artifacts.mli | 2 ++ src/dune_rules/gen_rules.ml | 4 +--- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/dune_rules/artifacts.ml b/src/dune_rules/artifacts.ml index 1bd42d2499a..b4a3781eec0 100644 --- a/src/dune_rules/artifacts.ml +++ b/src/dune_rules/artifacts.ml @@ -2,7 +2,9 @@ open Import open Memo.O module Bin = struct - let local_bin p = Path.Build.relative p ".bin" + let bin_dir_basename = ".bin" + + let local_bin p = Path.Build.relative p bin_dir_basename type t = { context : Context.t diff --git a/src/dune_rules/artifacts.mli b/src/dune_rules/artifacts.mli index 5cb29c56319..40d8aceeab5 100644 --- a/src/dune_rules/artifacts.mli +++ b/src/dune_rules/artifacts.mli @@ -3,6 +3,8 @@ open Import module Bin : sig type t + val bin_dir_basename : Filename.t + (** [local_bin dir] The directory which contains the local binaries viewed by rules defined in [dir] *) val local_bin : Path.Build.t -> Path.Build.t diff --git a/src/dune_rules/gen_rules.ml b/src/dune_rules/gen_rules.ml index 579c69e1dae..b1140493781 100644 --- a/src/dune_rules/gen_rules.ml +++ b/src/dune_rules/gen_rules.ml @@ -342,13 +342,11 @@ type automatic_sub_dir = | Formatted | Bin -let bin_dir_basename = ".bin" - let automatic_sub_dirs_map = String.Map.of_list_exn [ (Utop.utop_dir_basename, Utop) ; (Format_rules.formatted_dir_basename, Formatted) - ; (bin_dir_basename, Bin) + ; (Artifacts.Bin.bin_dir_basename, Bin) ] let gen_rules_for_automatic_sub_dir ~sctx ~dir kind = From 3773e7dae6a6a5e20323f6ccdf4c6b5c903e769d Mon Sep 17 00:00:00 2001 From: Antonio Nuno Monteiro Date: Sat, 26 Nov 2022 19:52:02 -0800 Subject: [PATCH 12/20] refactor(dune-rpc-lwt): forward compatiblity with upcoming Lwt_result (#6583) refactor(dune-rpc-lwt): forward compatibility forward compatiblity with upcoming Lwt_result Signed-off-by: Antonio Nuno Monteiro --- otherlibs/dune-rpc-lwt/src/dune_rpc_lwt.ml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/otherlibs/dune-rpc-lwt/src/dune_rpc_lwt.ml b/otherlibs/dune-rpc-lwt/src/dune_rpc_lwt.ml index b087660988d..f18601e24fc 100644 --- a/otherlibs/dune-rpc-lwt/src/dune_rpc_lwt.ml +++ b/otherlibs/dune-rpc-lwt/src/dune_rpc_lwt.ml @@ -85,7 +85,10 @@ module V1 = struct (Fiber) (struct let read_file s : (string, exn) result Lwt.t = - Lwt_result.catch (Lwt_io.with_file ~mode:Input s Lwt_io.read) + Lwt.catch + (fun () -> + Lwt_result.ok (Lwt_io.with_file ~mode:Input s Lwt_io.read)) + Lwt_result.fail let analyze_path s = Lwt.try_bind From 767b49a22e61d5559811cc71c60f5787f0e11036 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Sat, 26 Nov 2022 22:56:14 -0500 Subject: [PATCH 13/20] refactor: wrapped modules refactor (#6581) the library interface for modules shouuld always be present Signed-off-by: Rudi Grinberg --- src/dune_rules/module_compilation.ml | 11 ++++++----- src/dune_rules/modules.ml | 15 +++++++++------ 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/dune_rules/module_compilation.ml b/src/dune_rules/module_compilation.ml index ee9b49c6de2..bfa648f809e 100644 --- a/src/dune_rules/module_compilation.ml +++ b/src/dune_rules/module_compilation.ml @@ -408,14 +408,15 @@ module Alias_module = struct { local_name; obj_name }) in let shadowed = - if - Dune_project.dune_version project < (3, 5) - || Modules.lib_interface modules = None - then [] + if Dune_project.dune_version project < (3, 5) then [] else match Modules.alias_module modules with | None -> [] - | Some alias_module -> [ Module.name alias_module ] + | Some alias_module -> ( + match Modules.lib_interface modules with + | None -> [] + | Some m -> + if Module.kind m = Alias then [] else [ Module.name alias_module ]) in { main_module; aliases; shadowed } end diff --git a/src/dune_rules/modules.ml b/src/dune_rules/modules.ml index 59a1e2137d1..9a908855f6a 100644 --- a/src/dune_rules/modules.ml +++ b/src/dune_rules/modules.ml @@ -343,7 +343,9 @@ module Wrapped = struct || Module_name.Map.exists modules ~f || Module_name.Map.exists wrapped_compat ~f - let lib_interface t = Module_name.Map.find t.modules t.main_module_name + let lib_interface t = + Module_name.Map.find t.modules t.main_module_name + |> Option.value ~default:t.alias_module let find t name = if is_alias_name t name then Some t.alias_module @@ -355,6 +357,9 @@ module Wrapped = struct let find_dep t ~of_ name = match Module.kind of_ with | Alias -> None + | Wrapped_compat -> + let li = lib_interface t in + Option.some_if (name = Module.name li) li | _ -> if is_alias_name t name then Some t.alias_module else Module_name.Map.find t.modules name @@ -431,7 +436,7 @@ and dyn_of_impl { impl; vlib } = let rec lib_interface = function | Singleton m -> Some m | Unwrapped _ -> None - | Wrapped w -> Wrapped.lib_interface w + | Wrapped w -> Some (Wrapped.lib_interface w) | Stdlib w -> Stdlib.lib_interface w | Impl { impl = _; vlib } -> lib_interface vlib @@ -718,11 +723,9 @@ let entry_modules t = | Stdlib w -> Stdlib.lib_interface w |> Option.to_list | Singleton m -> [ m ] | Unwrapped m -> Module_name.Map.values m - | Wrapped m -> ( + | Wrapped m -> (* we assume this is never called for implementations *) - match Wrapped.lib_interface m with - | Some m -> [ m ] - | None -> [ m.alias_module ]) + [ Wrapped.lib_interface m ] | Impl i -> Code_error.raise "entry_modules: not defined for implementations" [ ("impl", dyn_of_impl i) ]) From d20daf278d747fb0f9dff557895bef150db8aab8 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Sun, 27 Nov 2022 11:51:26 -0500 Subject: [PATCH 14/20] chore: fix makefile (#6590) The last leading backslash isn't needed Signed-off-by: Rudi Grinberg --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 9da2ecfbb4e..f1dde5bb50b 100644 --- a/Makefile +++ b/Makefile @@ -27,7 +27,7 @@ result \ ctypes \ "utop>=2.6.0" \ "melange>=0.3.0" \ -"mel>=0.3.0" \ +"mel>=0.3.0" # Dependencies recommended for developing dune locally, # but not wanted in CI From 32b7864443c74b8107bd22c37cfa0bdbadedfde8 Mon Sep 17 00:00:00 2001 From: Antonio Nuno Monteiro Date: Sun, 27 Nov 2022 08:53:36 -0800 Subject: [PATCH 15/20] fix(rule generation): redirect to melange.emit parents (#6579) Correctly handle directories that are under a melange.emit's target Signed-off-by: Antonio Nuno Monteiro --- src/dune_rules/gen_rules.ml | 43 ++++++++++++++++--- .../melange/empty-entries-target-dir-exists.t | 5 +-- 2 files changed, 38 insertions(+), 10 deletions(-) diff --git a/src/dune_rules/gen_rules.ml b/src/dune_rules/gen_rules.ml index b1140493781..a7d8e971c7f 100644 --- a/src/dune_rules/gen_rules.ml +++ b/src/dune_rules/gen_rules.ml @@ -372,6 +372,25 @@ let has_rules subdirs f = ; rules }) +let rec has_melange_emit_parent dir = + match Path.Build.parent dir with + | None -> Memo.return false + | Some parent_dir -> ( + Only_packages.stanzas_in_dir parent_dir >>= function + | None -> has_melange_emit_parent parent_dir + | Some { Dune_file.stanzas; dir = _; project = _ } -> ( + match + List.exists stanzas ~f:(function + | Melange_emit mel -> + (* current dir, e.g. `_build/default/lib/output must be equal to + melange.emit target, e.g. `_build/default/lib` with + `(melange.emit (target output))` *) + Path.Build.equal (Path.Build.relative parent_dir mel.target) dir + | _ -> false) + with + | true -> Memo.return true + | false -> has_melange_emit_parent parent_dir)) + let redirect_to_parent = Memo.return Build_config.Redirect_to_parent (* Once [gen_rules] has decided what to do with the directory, it should end @@ -465,14 +484,24 @@ let gen_rules ~sctx ~dir components : Build_config.gen_rules_result Memo.t = (String.Set.of_list [ ".js"; "_doc"; ".ppx"; ".dune"; ".topmod" ]) | _ -> subdirs in - let+ directory_targets = + let* directory_targets = collect_directory_targets ~dir ~init:directory_targets - in - Build_config.Rules - { build_dir_only_sub_dirs = S.These subdirs - ; directory_targets - ; rules - })) + and* has_melange_emit_parent = has_melange_emit_parent dir in + + if has_melange_emit_parent then + (* The `melange.emit` stanza can generate rules in its sub-directories + if they declare melange libraries. As such, in the cases where a + parent declares a `melange.emit` stanza, this function needs to + augment the rules generated for the current directory with the + rules generated by `melange.emit` in the (possibly grand-)parent. *) + redirect_to_parent + else + Memo.return + (Build_config.Rules + { Build_config.build_dir_only_sub_dirs = S.These subdirs + ; directory_targets + ; rules + }))) let with_context ctx ~f = Super_context.find ctx >>= function diff --git a/test/blackbox-tests/test-cases/melange/empty-entries-target-dir-exists.t b/test/blackbox-tests/test-cases/melange/empty-entries-target-dir-exists.t index 3e46464365e..f34194cb0ef 100644 --- a/test/blackbox-tests/test-cases/melange/empty-entries-target-dir-exists.t +++ b/test/blackbox-tests/test-cases/melange/empty-entries-target-dir-exists.t @@ -17,6 +17,5 @@ Create the dist folder $ mkdir ./dist $ dune build @melange-dist - Error: No rule found for dist/melange.js - -> required by alias melange-dist - [1] + $ ls _build/default/dist + melange.js From 55bc124307133af8283fde90a8b146cbb016a7a7 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Sun, 27 Nov 2022 11:57:40 -0500 Subject: [PATCH 16/20] refactor: simplify by passing alias module (#6588) Signed-off-by: Rudi Grinberg --- src/dune_rules/module_compilation.ml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/dune_rules/module_compilation.ml b/src/dune_rules/module_compilation.ml index bfa648f809e..304b6a08f60 100644 --- a/src/dune_rules/module_compilation.ml +++ b/src/dune_rules/module_compilation.ml @@ -399,7 +399,7 @@ module Alias_module = struct (Module_name.to_string shadowed)); Buffer.contents b - let of_modules project modules = + let of_modules project modules ~alias_module = let main_module = Modules.main_module_name modules |> Option.value_exn in let aliases = Modules.for_alias modules @@ -410,13 +410,10 @@ module Alias_module = struct let shadowed = if Dune_project.dune_version project < (3, 5) then [] else - match Modules.alias_module modules with + match Modules.lib_interface modules with | None -> [] - | Some alias_module -> ( - match Modules.lib_interface modules with - | None -> [] - | Some m -> - if Module.kind m = Alias then [] else [ Module.name alias_module ]) + | Some m -> + if Module.kind m = Alias then [] else [ Module.name alias_module ] in { main_module; aliases; shadowed } end @@ -425,7 +422,7 @@ let build_alias_module cctx alias_module = let modules = Compilation_context.modules cctx in let alias_file () = let project = Compilation_context.scope cctx |> Scope.project in - Alias_module.of_modules project modules |> Alias_module.to_ml + Alias_module.of_modules project modules ~alias_module |> Alias_module.to_ml in let cctx = Compilation_context.for_alias_module cctx alias_module in let sctx = Compilation_context.super_context cctx in From 91265d53df39a5a082d14e1b4278aa7ebd0edc50 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Sun, 27 Nov 2022 11:58:21 -0500 Subject: [PATCH 17/20] refactor: wrap modules in a single place (#6589) Signed-off-by: Rudi Grinberg --- src/dune_rules/modules.ml | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/src/dune_rules/modules.ml b/src/dune_rules/modules.ml index 9a908855f6a..9901a226790 100644 --- a/src/dune_rules/modules.ml +++ b/src/dune_rules/modules.ml @@ -188,6 +188,21 @@ module Mangle = struct | _ -> prefix.public in Module.generated ~kind:Alias ~src_dir name + + let wrap_modules t modules = + let prefix = prefix t in + let f = + match t with + | Exe | Melange -> Module.with_wrapper ~main_module_name:prefix.public + | Lib { main_module_name; kind = _ } -> + fun m -> + if Module.name m = main_module_name then m + else + let visibility = Module.visibility m in + let prefix = Visibility.Map.find prefix visibility in + Module.with_wrapper m ~main_module_name:prefix + in + Module_name.Map.map modules ~f end let impl_only_of_map m = @@ -239,21 +254,12 @@ module Wrapped = struct ; alias_module = f alias_module } - let wrap_modules prefix ~main_module_name ~modules = - Module_name.Map.map modules ~f:(fun (m : Module.t) -> - if Module.name m = main_module_name then m - else - let visibility = Module.visibility m in - let prefix = Visibility.Map.find prefix visibility in - Module.with_wrapper m ~main_module_name:prefix) - let make ~src_dir ~lib_name ~implements ~modules ~main_module_name ~wrapped = let mangle = Mangle.of_lib ~main_module_name ~lib_name ~implements ~modules in let modules, wrapped_compat = - let prefix = Mangle.prefix mangle in - let wrapped_modules = wrap_modules prefix ~main_module_name ~modules in + let wrapped_modules = Mangle.wrap_modules mangle modules in match (wrapped : Mode.t) with | Simple false -> assert false | Simple true -> (wrapped_modules, Module_name.Map.empty) @@ -269,12 +275,8 @@ module Wrapped = struct { modules; alias_module; wrapped_compat; main_module_name; wrapped } let make_exe_or_melange ~src_dir ~modules mangle = - let prefix = Mangle.prefix mangle in let alias_module = Mangle.make_alias_module mangle ~src_dir in - let modules = - Module_name.Map.map modules ~f:(fun m -> - Module.with_wrapper m ~main_module_name:prefix.public) - in + let modules = Mangle.wrap_modules mangle modules in { modules ; wrapped_compat = Module_name.Map.empty ; alias_module From 92963851fde1f2ef031a5da9def835696829d39e Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Sun, 27 Nov 2022 12:49:40 -0500 Subject: [PATCH 18/20] chore: remove result type dependency (#6591) Signed-off-by: Rudi Grinberg --- Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/Makefile b/Makefile index f1dde5bb50b..24c56e38202 100644 --- a/Makefile +++ b/Makefile @@ -23,7 +23,6 @@ ocamlformat.$$(awk -F = '$$1 == "version" {print $$2}' .ocamlformat) \ "ppx_expect.v0.15.0" \ ppx_inline_test \ ppxlib \ -result \ ctypes \ "utop>=2.6.0" \ "melange>=0.3.0" \ From 2131800f687cefffb4d139c17a31b93590836687 Mon Sep 17 00:00:00 2001 From: Antonio Nuno Monteiro Date: Sun, 27 Nov 2022 11:05:09 -0800 Subject: [PATCH 19/20] fix: `make docker-build-image` (#6592) Signed-off-by: Antonio Nuno Monteiro --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 24c56e38202..96ae2a01c04 100644 --- a/Makefile +++ b/Makefile @@ -25,8 +25,8 @@ ppx_inline_test \ ppxlib \ ctypes \ "utop>=2.6.0" \ -"melange>=0.3.0" \ -"mel>=0.3.0" +"melange>=0.3.1" \ +"mel>=0.3.1" # Dependencies recommended for developing dune locally, # but not wanted in CI From 5a098272172c1f16aad309d6aabc016f7be2a9f3 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Sun, 27 Nov 2022 17:53:26 -0500 Subject: [PATCH 20/20] refactor: remove Modules.map (#6595) it's unused Signed-off-by: Rudi Grinberg --- src/dune_rules/modules.ml | 8 -------- src/dune_rules/modules.mli | 2 -- 2 files changed, 10 deletions(-) diff --git a/src/dune_rules/modules.ml b/src/dune_rules/modules.ml index 9901a226790..1d73d4e9d73 100644 --- a/src/dune_rules/modules.ml +++ b/src/dune_rules/modules.ml @@ -563,14 +563,6 @@ let rec fold_no_vlib t ~init ~f = | Wrapped w -> Wrapped.fold w ~init ~f | Impl { vlib = _; impl } -> fold_no_vlib impl ~f ~init -let rec map t ~f = - match t with - | Stdlib w -> Stdlib (Stdlib.map w ~f) - | Singleton m -> Singleton (f m) - | Unwrapped m -> Unwrapped (Module_name.Map.map m ~f) - | Wrapped w -> Wrapped (Wrapped.map w ~f) - | Impl { vlib; impl } -> Impl { vlib = map vlib ~f; impl = map impl ~f } - type split_by_lib = { vlib : Module.t list ; impl : Module.t list diff --git a/src/dune_rules/modules.mli b/src/dune_rules/modules.mli index c658d2df32c..230d910c870 100644 --- a/src/dune_rules/modules.mli +++ b/src/dune_rules/modules.mli @@ -57,8 +57,6 @@ val fold_user_written : t -> f:(Module.t -> 'acc -> 'acc) -> init:'acc -> 'acc val map_user_written : t -> f:(Module.t -> Module.t Memo.t) -> t Memo.t -val map : t -> f:(Module.t -> Module.t) -> t - val fold_user_available : t -> f:(Module.t -> 'acc -> 'acc) -> init:'acc -> 'acc (** Returns all the compatibility modules. *)