feat(melange): install melange libraries (#6602)

Signed-off-by: Antonio Nuno Monteiro <[email protected]>
Signed-off-by: Javier Chavarri <[email protected]>
Co-authored-by: Javier Chavarri <[email protected]>

src/dune_rules/dune_package.ml

@@ -90,7 +90,7 @@ module Lib = struct
        ; field_o "default_implementation" (no_loc Lib_name.encode)
            default_implementation
        ; field_o "main_module_name" Module_name.encode main_module_name
-       ; field_l "modes" sexp (Mode.Dict.Set.encode modes.ocaml)
+       ; field_l "modes" sexp (Lib_mode.Map.Set.encode modes)
        ; field_l "obj_dir" sexp (Obj_dir.encode obj_dir)
        ; field_o "modules" Modules.encode modules
        ; field_o "special_builtin_support"
@@ -131,7 +131,7 @@ module Lib = struct
        in
        let+ synopsis = field_o "synopsis" string
        and+ loc = loc
-       and+ modes = field_l "modes" Mode.decode
+       and+ modes = field_l "modes" Lib_mode.decode
        and+ kind = field "kind" Lib_kind.decode
        and+ archives = mode_paths "archives"
        and+ plugins = mode_paths "plugins"
@@ -165,7 +165,7 @@ module Lib = struct
        and+ instrumentation_backend =
          field_o "instrumentation.backend" (located Lib_name.decode)
        in
-       let modes = Mode.Dict.Set.of_list modes in
+       let modes = Lib_mode.Map.Set.of_list modes in
        let entry_modules =
          Modules.entry_modules modules |> List.map ~f:Module.name
        in
@@ -190,7 +190,6 @@ module Lib = struct
            Some (Lib_info.Inherited.This (Modules.wrapped modules))
          in
          let entry_modules = Lib_info.Source.External (Ok entry_modules) in
-         let modes = { Lib_mode.Map.ocaml = modes; melange = false } in
          let modules = Lib_info.Source.External (Some modules) in
          Lib_info.create ~path_kind:External ~loc ~name ~kind ~status ~src_dir
            ~orig_src_dir ~obj_dir ~version ~synopsis ~main_module_name

src/dune_rules/install_rules.ml

@@ -134,8 +134,8 @@ end = struct
               make_entry Lib source ?dst))
     in
     let { Lib_config.has_native; ext_obj; _ } = lib_config in
-    let modes = Dune_file.Mode_conf.Set.eval lib.modes.ocaml ~has_native in
-    let { Mode.Dict.byte; native } = modes in
+    let modes = Dune_file.Mode_conf.Lib.Set.eval lib.modes ~has_native in
+    let { Lib_mode.Map.ocaml = { Mode.Dict.byte; native }; melange } = modes in
     let module_files =
       let inside_subdir f =
         match lib_subdir with
@@ -147,30 +147,30 @@ end = struct
       in
       let cm_dir m cm_kind =
         let visibility = Module.visibility m in
-        let dir' = Obj_dir.cm_dir external_obj_dir (Ocaml cm_kind) visibility in
+        let dir' = Obj_dir.cm_dir external_obj_dir cm_kind visibility in
         if Path.equal (Path.build dir) dir' then None
         else Path.basename dir' |> inside_subdir |> Option.some
       in
       let virtual_library = Library.is_virtual lib in
+      let if_ b (cm_kind, f) =
+        if b then
+          match f with
+          | None -> []
+          | Some f -> [ (cm_kind, f) ]
+        else []
+      in
       let modules =
         let common m =
-          let cm_file kind =
-            Obj_dir.Module.cm_file obj_dir m ~kind:(Ocaml kind)
-          in
-          let if_ b (cm_kind, f) =
-            if b then
-              match f with
-              | None -> []
-              | Some f -> [ (cm_kind, f) ]
-            else []
-          in
-          let open Cm_kind in
-          [ if_ true (Cmi, cm_file Cmi)
-          ; if_ native (Cmx, cm_file Cmx)
-          ; if_ (byte && virtual_library) (Cmo, cm_file Cmo)
+          let cm_file kind = Obj_dir.Module.cm_file obj_dir m ~kind in
+          let open Lib_mode.Cm_kind in
+          [ if_ (native || byte) (Ocaml Cmi, cm_file (Ocaml Cmi))
+          ; if_ native (Ocaml Cmx, cm_file (Ocaml Cmx))
+          ; if_ (byte && virtual_library) (Ocaml Cmo, cm_file (Ocaml Cmo))
           ; if_
               (native && virtual_library)
-              (Cmx, Obj_dir.Module.o_file obj_dir m ~ext_obj)
+              (Ocaml Cmx, Obj_dir.Module.o_file obj_dir m ~ext_obj)
+          ; if_ melange (Melange Cmi, cm_file (Melange Cmi))
+          ; if_ melange (Melange Cmj, cm_file (Melange Cmj))
           ]
           |> List.concat
         in
@@ -179,15 +179,19 @@ end = struct
         in
         let modules_impl =
           List.concat_map installable_modules.impl ~f:(fun m ->
-              common m
-              @ List.filter_map Ml_kind.all ~f:(fun ml_kind ->
-                    let open Option.O in
-                    let+ cmt =
-                      Obj_dir.Module.cmt_file obj_dir m ~ml_kind
-                        ~cm_kind:(Ocaml Cmi)
-                    in
-                    (Cm_kind.Cmi, cmt))
-              |> set_dir m)
+              let cmt_files =
+                List.concat_map Ml_kind.all ~f:(fun ml_kind ->
+                    let open Lib_mode.Cm_kind in
+                    List.concat_map
+                      [ (native || byte, Ocaml Cmi); (melange, Melange Cmi) ]
+                      ~f:(fun (condition, kind) ->
+                        if_ condition
+                          ( kind
+                          , Obj_dir.Module.cmt_file obj_dir m ~ml_kind
+                              ~cm_kind:kind )))
+              in
+
+              common m @ cmt_files |> set_dir m)
         in
         let modules_vlib =
           List.concat_map installable_modules.vlib ~f:(fun m ->
@@ -199,7 +203,10 @@ end = struct
     in
     let* lib_files, dll_files =
       let+ lib_files = lib_files ~dir ~dir_contents ~lib_config info in
-      let dll_files = dll_files ~modes ~dynlink:lib.dynlink ~ctx info in
+      let dll_files =
+        let modes = modes.ocaml in
+        dll_files ~modes ~dynlink:lib.dynlink ~ctx info
+      in
       (lib_files, dll_files)
     in
     let+ execs = lib_ppxs ctx ~scope ~lib in

src/dune_rules/lib_mode.ml

@@ -1,7 +1,28 @@
+open Stdune
+
 type t =
   | Ocaml of Ocaml.Mode.t
   | Melange
 
+let equal x y =
+  match (x, y) with
+  | Ocaml o1, Ocaml o2 -> Ocaml.Mode.equal o1 o2
+  | Ocaml _, _ | _, Ocaml _ -> false
+  | Melange, Melange -> true
+
+let decode =
+  let open Dune_sexp.Decoder in
+  enum [ ("byte", Ocaml Byte); ("native", Ocaml Native); ("melange", Melange) ]
+
+let choose byte native melange = function
+  | Ocaml Byte -> byte
+  | Ocaml Native -> native
+  | Melange -> melange
+
+let to_string = choose "byte" "native" "melange"
+
+let encode t = Dune_sexp.Encoder.string (to_string t)
+
 module Cm_kind = struct
   type t =
     | Ocaml of Ocaml.Cm_kind.t
@@ -51,6 +72,8 @@ let of_cm_kind : Cm_kind.t -> t = function
   | Melange (Cmi | Cmj) -> Melange
 
 module Map = struct
+  let mode_equal = equal
+
   type 'a t =
     { ocaml : 'a Ocaml.Mode.Dict.t
     ; melange : 'a
@@ -83,6 +106,23 @@ module Map = struct
 
     let equal = equal Bool.equal
 
+    let to_list (t : t) =
+      let l = [] in
+      let l = if t.ocaml.native then Ocaml Native :: l else l in
+      let l = if t.ocaml.byte then Ocaml Byte :: l else l in
+      let l = if t.melange then Melange :: l else l in
+      l
+
+    let encode t = List.map ~f:encode (to_list t)
+
+    let of_list l =
+      { ocaml =
+          { byte = List.mem l (Ocaml Byte) ~equal:mode_equal
+          ; native = List.mem l (Ocaml Native) ~equal:mode_equal
+          }
+      ; melange = List.mem l Melange ~equal:mode_equal
+      }
+
     let to_dyn { ocaml; melange } =
       let open Dyn in
       record

src/dune_rules/lib_mode.mli

@@ -2,6 +2,8 @@ type t =
   | Ocaml of Ocaml.Mode.t
   | Melange
 
+val decode : t Dune_sexp.Decoder.t
+
 module Cm_kind : sig
   type t =
     | Ocaml of Ocaml.Cm_kind.t
@@ -54,6 +56,10 @@ module Map : sig
   module Set : sig
     type nonrec t = bool t
 
+    val encode : t -> Dune_sexp.t list
+
+    val of_list : mode list -> t
+
     val to_dyn : t -> Dyn.t
 
     val equal : t -> t -> bool

src/dune_rules/melange.ml

@@ -39,6 +39,10 @@ module Cm_kind = struct
   end
 end
 
+module Install = struct
+  let dir = "melange"
+end
+
 let js_basename m =
   match Module.file ~ml_kind:Impl m with
   | Some s -> (

src/dune_rules/melange.mli

@@ -29,4 +29,8 @@ module Cm_kind : sig
   end
 end
 
+module Install : sig
+  val dir : string
+end
+
 val js_basename : Module.t -> Filename.t

src/dune_rules/melange_rules.ml

@@ -16,24 +16,37 @@ let ocaml_flags sctx ~dir melange =
     let ocaml_version = (Super_context.context sctx).version in
     Super_context.with_vendored_flags ~ocaml_version flags
 
-let lib_output_dir ~target_dir ~lib_dir =
+let lib_output_dir ~sctx ~target_dir lib =
+  let info = Lib.info lib in
+  let lib_dir =
+    match Lib_info.status info with
+    | Private _ ->
+      let lib = Lib.Local.of_lib_exn lib in
+      let info = Lib.Local.info lib in
+      Lib_info.src_dir info
+    | Public _ ->
+      let package_name = Option.value_exn (Lib_info.package info) in
+      let bctx = (Super_context.context sctx).build_dir in
+      let info = Lib.info lib in
+      let src_dir = Lib_info.src_dir info in
+      Path.Build.L.relative bctx
+        [ "node_modules"
+        ; Package.Name.to_string package_name
+        ; Path.Source.to_string (Path.drop_build_context_exn src_dir)
+        ]
+    | Installed | Installed_private ->
+      let package_name = Option.value_exn (Lib_info.package info) in
+      let bctx = (Super_context.context sctx).build_dir in
+      Path.Build.L.relative bctx
+        [ "node_modules"; Package.Name.to_string package_name ]
+  in
   Path.Build.append_source target_dir
     (Path.Build.drop_build_context_exn lib_dir)
 
 let make_js_name ~js_ext ~dst_dir m =
   let name = Melange.js_basename m ^ js_ext in
   Path.Build.relative dst_dir name
 
-let local_of_lib ~loc lib =
-  match Lib.Local.of_lib lib with
-  | Some s -> s
-  | None ->
-    let lib_name = Lib.name lib in
-    User_error.raise ~loc
-      [ Pp.textf "The external library %s cannot be used"
-          (Lib_name.to_string lib_name)
-      ]
-
 let impl_only_modules_defined_in_this_lib sctx lib =
   let open Memo.O in
   let+ modules = Dir_contents.modules_of_lib sctx lib >>| Option.value_exn in
@@ -82,12 +95,11 @@ let js_targets_of_modules modules ~js_ext ~dst_dir =
         Path.Set.add acc target
       else acc)
 
-let js_targets_of_libs sctx libs ~js_ext ~loc ~target_dir =
+let js_targets_of_libs sctx libs ~js_ext ~target_dir =
   let of_lib lib =
     let open Memo.O in
     let+ modules = impl_only_modules_defined_in_this_lib sctx lib in
-    let lib_dir = local_of_lib ~loc lib |> Lib.Local.info |> Lib_info.src_dir in
-    let dst_dir = lib_output_dir ~target_dir ~lib_dir in
+    let dst_dir = lib_output_dir ~sctx ~target_dir lib in
     List.rev_map modules ~f:(fun m ->
         Path.build @@ make_js_name ~js_ext ~dst_dir m)
   in
@@ -108,9 +120,7 @@ let build_js ~loc ~dir ~pkg_name ~mode ~module_system ~dst_dir ~obj_dir ~sctx
   let* compiler = Melange_binary.melc sctx ~loc:(Some loc) ~dir in
   let src = Obj_dir.Module.cm_file_exn obj_dir m ~kind:(Melange Cmj) in
   let output = make_js_name ~js_ext ~dst_dir m in
-  let obj_dir =
-    [ Command.Args.A "-I"; Path (Path.build (Obj_dir.melange_dir obj_dir)) ]
-  in
+  let obj_dir = [ Command.Args.A "-I"; Path (Obj_dir.melange_dir obj_dir) ] in
   let melange_package_args =
     let pkg_name_args =
       match pkg_name with
@@ -130,7 +140,7 @@ let build_js ~loc ~dir ~pkg_name ~mode ~module_system ~dst_dir ~obj_dir ~sctx
        ; As melange_package_args
        ; A "-o"
        ; Target output
-       ; Dep (Path.build src)
+       ; Dep src
        ])
 
 let setup_emit_cmj_rules ~sctx ~dir ~scope ~expander ~dir_contents
@@ -202,7 +212,7 @@ let setup_emit_cmj_rules ~sctx ~dir ~scope ~expander ~dir_contents
           @@
           let open Resolve.Memo.O in
           Compilation_context.requires_link cctx
-          >>= js_targets_of_libs sctx ~js_ext ~loc:mel.loc ~target_dir
+          >>= js_targets_of_libs sctx ~js_ext ~target_dir
         in
         Action_builder.paths deps)
         |> Rules.Produce.Alias.add_deps alias
@@ -253,6 +263,7 @@ let setup_entries_js ~sctx ~dir ~dir_contents ~scope ~compile_info ~target_dir
       (Path.Build.drop_build_context_exn (Dir_contents.dir dir_contents))
   in
   Memo.parallel_iter modules_for_js ~f:(fun m ->
+      let obj_dir = Obj_dir.of_local obj_dir in
       build_js ~dir ~loc ~pkg_name ~mode ~module_system:mel.module_system
         ~dst_dir ~obj_dir ~sctx ~includes ~js_ext m)
 
@@ -266,7 +277,7 @@ let setup_js_rules_libraries ~dir ~scope ~target_dir ~sctx ~requires_link ~mode
         Lib.DB.get_compile_info (Scope.libs scope) lib_name
           ~allow_overlaps:mel.allow_overlapping_dependencies
       in
-      let info = local_of_lib ~loc:mel.loc lib |> Lib.Local.info in
+      let info = Lib.info lib in
       let loc = Lib_info.loc info in
       let build_js =
         let obj_dir = Lib_info.obj_dir info in
@@ -284,12 +295,7 @@ let setup_js_rules_libraries ~dir ~scope ~target_dir ~sctx ~requires_link ~mode
         | None -> Memo.return ()
         | Some vlib ->
           let* vlib = Resolve.Memo.read_memo vlib in
-          let dst_dir =
-            let lib_dir =
-              local_of_lib ~loc vlib |> Lib.Local.info |> Lib_info.src_dir
-            in
-            lib_output_dir ~target_dir ~lib_dir
-          in
+          let dst_dir = lib_output_dir ~sctx ~target_dir vlib in
           let* includes =
             let+ requires_link =
               Lib.Compile.for_lib
@@ -303,10 +309,7 @@ let setup_js_rules_libraries ~dir ~scope ~target_dir ~sctx ~requires_link ~mode
           >>= Memo.parallel_iter ~f:(build_js ~dir ~dst_dir ~includes)
       in
       let* source_modules = impl_only_modules_defined_in_this_lib sctx lib in
-      let dst_dir =
-        let lib_dir = Lib_info.src_dir info in
-        lib_output_dir ~target_dir ~lib_dir
-      in
+      let dst_dir = lib_output_dir ~sctx ~target_dir lib in
       Memo.parallel_iter source_modules ~f:(build_js ~dir ~dst_dir ~includes))
 
 let setup_emit_js_rules ~dir_contents ~dir ~scope ~sctx mel =

src/dune_rules/obj_dir.ml

@@ -45,8 +45,9 @@ module External = struct
       if private_lib then Some (Path.relative dir ".public_cmi") else None
     in
     let public_cmi_melange_dir =
-      if private_lib then Some (Path.relative dir ".public_cmi_melange")
-      else None
+      let melange_dir = Path.relative dir Melange.Install.dir in
+      if private_lib then Some (Path.relative melange_dir ".public_cmi_melange")
+      else Some melange_dir
     in
     { public_dir = dir
     ; private_dir
@@ -78,7 +79,8 @@ module External = struct
       Code_error.raise "External.cm_dir" [ ("t", to_dyn t) ]
     | Ocaml Cmi, Public, _ -> public_cmi_ocaml_dir t
     | Melange Cmi, Public, _ -> public_cmi_melange_dir t
-    | (Ocaml (Cmo | Cmx) | Melange Cmj), _, _ -> t.public_dir
+    | Melange Cmj, _, _ -> public_cmi_melange_dir t
+    | Ocaml (Cmo | Cmx), _, _ -> t.public_dir
 
   let encode
       { public_dir