Improve js_of_ocaml support

CHANGES.md

@@ -1,6 +1,24 @@
 Unreleased
 ----------
 
+- Run tests in all modes defined. Previously, jsoo was excluded. (@hhugo,
+  #5049, fix #4951)
+
+- Allow to configure the alias to run the jsoo tests (@hhugo, #5049, #4999)
+
+- Set jsoo compilation flags in the `env` stanza (@hhugo, #5049, #1613)
+
+- Allow to configure jsoo separate compilation in the `env` stanza. Previously,
+  it was hard coded to always be enabled in the `dev` profile. (@hhugo, #5049,
+  fix #970)
+
+- Fix build-info version in jsoo executables (@hhugo, #5049, fix #4444)
+
+- Pass `-no-check-prims` when building bytecode for jsoo (@hhugo, #5049, #4027)
+
+- Fix jsoo builds when dynamically linked foreign archives are disabled
+  (@hhugo, #5049)
+
 - Warn on empty packages for projects < 3.0 and disallow them starting from 3.0
   Empty packages may be re-enabled by adding the `(allow_empty)` to the package
   stanza in the dune-project file. (#4867, fix #2882, @kit-ty-kate, @rgrinberg)

doc/dune-files.rst

@@ -663,7 +663,13 @@ options using ``(js_of_ocaml (<js_of_ocaml-options>))``.
 
 ``<js_of_ocaml-options>`` are all optional:
 
-- ``(flags <flags>)`` to specify flags passed to ``js_of_ocaml``. This field
+- ``(flags <flags>)`` to specify flags passed to ``js_of_ocaml compile``. This field
+  supports ``(:include ...)`` forms
+
+- ``(build_runtime_flags <flags>)`` to specify flags passed to ``js_of_ocaml build-runtime``. This field
+  supports ``(:include ...)`` forms
+
+- ``(link_flags <flags>)`` to specify flags passed to ``js_of_ocaml link``. This field
   supports ``(:include ...)`` forms
 
 - ``(javascript_files (<files-list>))`` to specify ``js_of_ocaml`` JavaScript
@@ -1524,6 +1530,17 @@ Fields supported in ``<settings>`` are:
 
 - ``(menhir_flags <flags>))`` specifies flags for Menhir stanzas.
 
+- ``(js_of_ocaml (flags <flags>)(build_runtime <flags>)(link_flags <flags>))``
+  to specify js_of_ocaml flags. see `jsoo-field`_ for more details.
+
+- ``(js_of_ocaml (compilation_mode <mode>))``, where ``<mode>`` is
+  either ``whole_program`` or ``separate``. This field controls
+  whether to use separate compilation or not.
+
+- ``(js_of_ocaml (runtest_alias <alias-name>))`` is used to specify
+  the alias under which `inline_tests`_ and tests (`tests-stanza`_)
+  run for the `js` mode.
+
 - ``(binaries <binaries>)``, where ``<binaries>`` is a list of entries
   of the form ``(<filepath> as <name>)``. ``(<filepath> as <name>)``
   makes the binary ``<filepath>`` available in the command search as

doc/jsoo.rst

@@ -65,8 +65,8 @@ Dune supports two modes of compilation
   separately and then linked together. This mode is useful during development as
   it builds more quickly.
 
-The separate compilation mode will be selected when the build profile is
-``dev``, which is the default. There is currently no other way to control this
-behaviour.
+The separate compilation mode will be selected when the build profile
+is ``dev``, which is the default. It can also be explicitly sepcified
+in an env stanza. See :ref:`dune-env` for more information.
 
 .. _js_of_ocaml: http://ocsigen.org/js_of_ocaml/

otherlibs/build-info/test/run.t

@@ -94,6 +94,7 @@ Check what the generated build info module looks like:
   $ cat _build/default/c/.c.eobjs/build_info_data.ml-gen \
   >   | sed 's/"dune-build-info".*/"dune-build-info", Some "XXX"/'
   let eval s =
+    let s = Bytes.unsafe_to_string (Bytes.unsafe_of_string s) in
     let len = String.length s in
     if s.[0] = '=' then
       let colon_pos = String.index_from s 1 ':' in
@@ -106,9 +107,9 @@ Check what the generated build info module looks like:
       None
   [@@inline never]
 
-  let p1 = eval (Sys.opaque_identity "%%DUNE_PLACEHOLDER:64:vcs-describe:1:a%%%%%%%%%%%%%%%%%%%%%%%%%%")
-  let p2 = eval (Sys.opaque_identity "%%DUNE_PLACEHOLDER:64:vcs-describe:1:b%%%%%%%%%%%%%%%%%%%%%%%%%%")
-  let p0 = eval (Sys.opaque_identity "%%DUNE_PLACEHOLDER:64:vcs-describe:1:c%%%%%%%%%%%%%%%%%%%%%%%%%%")
+  let p1 = eval "%%DUNE_PLACEHOLDER:64:vcs-describe:1:a%%%%%%%%%%%%%%%%%%%%%%%%%%"
+  let p2 = eval "%%DUNE_PLACEHOLDER:64:vcs-describe:1:b%%%%%%%%%%%%%%%%%%%%%%%%%%"
+  let p0 = eval "%%DUNE_PLACEHOLDER:64:vcs-describe:1:c%%%%%%%%%%%%%%%%%%%%%%%%%%"
 
   let version = p0
 

src/dune_engine/alias.ml

@@ -160,6 +160,10 @@ let make_standard name =
   Table.add_exn standard_aliases name ();
   make name
 
+let register_as_standard name =
+  let (_ : (unit, _) result) = Table.add standard_aliases name () in
+  ()
+
 let default = make_standard Name.default
 
 let runtest = make_standard Name.runtest

src/dune_engine/alias.mli

@@ -7,6 +7,8 @@ module Name : sig
 
   val of_string : string -> t
 
+  val equal : t -> t -> bool
+
   val parse_string_exn : Loc.t * string -> t
 
   val to_string : t -> string
@@ -38,6 +40,8 @@ val compare : t -> t -> Ordering.t
 
 val make : Name.t -> dir:Path.Build.t -> t
 
+val register_as_standard : Name.t -> unit
+
 (** The following always holds: [make (name t) ~dir:(dir t) = t] *)
 val name : t -> Name.t
 

src/dune_rules/compilation_context.ml

@@ -66,7 +66,7 @@ type t =
   ; preprocessing : Pp_spec.t
   ; opaque : bool
   ; stdlib : Ocaml_stdlib.t option
-  ; js_of_ocaml : Dune_file.Js_of_ocaml.t option
+  ; js_of_ocaml : Js_of_ocaml.In_buildable.t option
   ; sandbox : Sandbox_config.t
   ; package : Package.t option
   ; vimpl : Vimpl.t option
@@ -212,7 +212,6 @@ let for_module_generated_at_link_time cctx ~requires ~module_ =
   let modules = Modules.singleton_exe module_ in
   { cctx with
     opaque
-  ; js_of_ocaml = None
   ; flags = Ocaml_flags.empty
   ; requires_link = Memo.lazy_ (fun () -> requires)
   ; requires_compile = requires

src/dune_rules/compilation_context.mli

@@ -34,7 +34,7 @@ val create :
   -> ?preprocessing:Pp_spec.t
   -> opaque:opaque
   -> ?stdlib:Ocaml_stdlib.t
-  -> js_of_ocaml:Dune_file.Js_of_ocaml.t option
+  -> js_of_ocaml:Js_of_ocaml.In_buildable.t option
   -> package:Package.t option
   -> ?vimpl:Vimpl.t
   -> ?modes:Dune_file.Mode_conf.Set.Details.t Mode.Dict.t
@@ -75,7 +75,7 @@ val opaque : t -> bool
 
 val stdlib : t -> Ocaml_stdlib.t option
 
-val js_of_ocaml : t -> Dune_file.Js_of_ocaml.t option
+val js_of_ocaml : t -> Js_of_ocaml.In_buildable.t option
 
 val sandbox : t -> Sandbox_config.t
 

src/dune_rules/dune_env.ml

@@ -79,6 +79,7 @@ module Stanza = struct
     ; inline_tests : Inline_tests.t option
     ; menhir_flags : Ordered_set_lang.Unexpanded.t
     ; odoc : Odoc.t
+    ; js_of_ocaml : Ordered_set_lang.Unexpanded.t Js_of_ocaml.Env.t
     ; coq : Ordered_set_lang.Unexpanded.t
     ; format_config : Format_config.t option
     }
@@ -91,6 +92,7 @@ module Stanza = struct
       ; inline_tests
       ; menhir_flags
       ; odoc
+      ; js_of_ocaml
       ; coq
       ; format_config
       } t =
@@ -104,6 +106,7 @@ module Stanza = struct
     && Odoc.equal odoc t.odoc
     && Ordered_set_lang.Unexpanded.equal coq t.coq
     && Option.equal Format_config.equal format_config t.format_config
+    && Js_of_ocaml.Env.equal js_of_ocaml t.js_of_ocaml
 
   let hash_config = Hashtbl.hash
 
@@ -116,6 +119,7 @@ module Stanza = struct
     ; inline_tests = None
     ; menhir_flags = Ordered_set_lang.Unexpanded.standard
     ; odoc = Odoc.empty
+    ; js_of_ocaml = Js_of_ocaml.Env.empty
     ; coq = Ordered_set_lang.Unexpanded.standard
     ; format_config = None
     }
@@ -164,6 +168,10 @@ module Stanza = struct
     field "odoc" ~default:Odoc.empty
       (Dune_lang.Syntax.since Stanza.syntax (2, 4) >>> Odoc.decode)
 
+  let js_of_ocaml_field =
+    field "js_of_ocaml" ~default:Js_of_ocaml.Env.empty
+      (Dune_lang.Syntax.since Stanza.syntax (3, 0) >>> Js_of_ocaml.Env.decode)
+
   let coq_flags = Ordered_set_lang.Unexpanded.field "flags"
 
   let coq_field =
@@ -181,6 +189,7 @@ module Stanza = struct
     and+ inline_tests = inline_tests_field
     and+ menhir_flags = menhir_flags ~since:(Some (2, 1))
     and+ odoc = odoc_field
+    and+ js_of_ocaml = js_of_ocaml_field
     and+ coq = coq_field
     and+ format_config = Format_config.field ~since:(2, 8) in
     { flags
@@ -190,6 +199,7 @@ module Stanza = struct
     ; inline_tests
     ; menhir_flags
     ; odoc
+    ; js_of_ocaml
     ; coq
     ; format_config
     }

src/dune_rules/dune_env.mli

@@ -33,6 +33,7 @@ module Stanza : sig
     ; inline_tests : Inline_tests.t option
     ; menhir_flags : Ordered_set_lang.Unexpanded.t
     ; odoc : Odoc.t
+    ; js_of_ocaml : Ordered_set_lang.Unexpanded.t Js_of_ocaml.Env.t
     ; coq : Ordered_set_lang.Unexpanded.t
     ; format_config : Format_config.t option
     }

src/dune_rules/dune_file.ml

@@ -27,24 +27,6 @@ module Lint = struct
   let no_lint = default
 end
 
-module Js_of_ocaml = struct
-  type t =
-    { flags : Ordered_set_lang.Unexpanded.t
-    ; javascript_files : string list
-    }
-
-  let decode =
-    fields
-      (let+ flags = Ordered_set_lang.Unexpanded.field "flags"
-       and+ javascript_files =
-         field "javascript_files" (repeat string) ~default:[]
-       in
-       { flags; javascript_files })
-
-  let default =
-    { flags = Ordered_set_lang.Unexpanded.standard; javascript_files = [] }
-end
-
 type for_ =
   | Executable
   | Library of Wrapped.t option
@@ -160,7 +142,7 @@ module Buildable = struct
     ; preprocessor_deps : Dep_conf.t list
     ; lint : Preprocess.Without_instrumentation.t Preprocess.Per_module.t
     ; flags : Ocaml_flags.Spec.t
-    ; js_of_ocaml : Js_of_ocaml.t
+    ; js_of_ocaml : Js_of_ocaml.In_buildable.t
     ; allow_overlapping_dependencies : bool
     ; ctypes : Ctypes_stanza.t option
     ; root_module : (Loc.t * Module_name.t) option
@@ -225,7 +207,8 @@ module Buildable = struct
     and+ libraries = field "libraries" (Lib_deps.decode for_) ~default:[]
     and+ flags = Ocaml_flags.Spec.decode
     and+ js_of_ocaml =
-      field "js_of_ocaml" Js_of_ocaml.decode ~default:Js_of_ocaml.default
+      field "js_of_ocaml" Js_of_ocaml.In_buildable.decode
+        ~default:Js_of_ocaml.In_buildable.default
     and+ allow_overlapping_dependencies =
       field_b "allow_overlapping_dependencies"
     and+ version = Dune_lang.Syntax.get_exn Stanza.syntax
@@ -1373,7 +1356,11 @@ module Executables = struct
         else
           singleton exe Loc.none
 
-      let default_for_tests = byte_and_exe
+      let default_for_tests ~version =
+        if version < (3, 0) then
+          byte_and_exe
+        else
+          singleton exe Loc.none
 
       let best_install_mode t = List.find ~f:(mem t) installable_modes
     end
@@ -1946,7 +1933,9 @@ module Tests = struct
             field "locks" (repeat String_with_vars.decode) ~default:[]
           and+ modes =
             field "modes" Executables.Link_mode.Map.decode
-              ~default:Executables.Link_mode.Map.default_for_tests
+              ~default:
+                (Executables.Link_mode.Map.default_for_tests
+                   ~version:dune_version)
           and+ enabled_if =
             Enabled_if.decode ~allowed_vars:Any ~since:(Some (1, 4)) ()
           and+ action =

src/dune_rules/dune_file.mli

@@ -10,15 +10,6 @@ module Lint : sig
   val no_lint : t
 end
 
-module Js_of_ocaml : sig
-  type t =
-    { flags : Ordered_set_lang.Unexpanded.t
-    ; javascript_files : string list
-    }
-
-  val default : t
-end
-
 type for_ =
   | Executable
   | Library of Wrapped.t option
@@ -50,7 +41,7 @@ module Buildable : sig
     ; preprocessor_deps : Dep_conf.t list
     ; lint : Lint.t
     ; flags : Ocaml_flags.Spec.t
-    ; js_of_ocaml : Js_of_ocaml.t
+    ; js_of_ocaml : Js_of_ocaml.In_buildable.t
     ; allow_overlapping_dependencies : bool
     ; ctypes : Ctypes_stanza.t option
     ; root_module : (Loc.t * Module_name.t) option

src/dune_rules/env_node.ml

@@ -23,6 +23,7 @@ type t =
   ; inline_tests : Dune_env.Stanza.Inline_tests.t Memo.Lazy.t
   ; menhir_flags : string list Action_builder.t Memo.Lazy.t
   ; odoc : Odoc.t Memo.Lazy.t
+  ; js_of_ocaml : string list Action_builder.t Js_of_ocaml.Env.t Memo.Lazy.t
   ; coq : Coq.t Action_builder.t Memo.Lazy.t
   ; format_config : Format_config.t Memo.Lazy.t
   }
@@ -41,6 +42,8 @@ let bin_artifacts t = Memo.Lazy.force t.bin_artifacts
 
 let inline_tests t = Memo.Lazy.force t.inline_tests
 
+let js_of_ocaml t = Memo.Lazy.force t.js_of_ocaml
+
 let menhir_flags t =
   Memo.Lazy.force t.menhir_flags |> Action_builder.memo_build_join
 
@@ -125,6 +128,28 @@ let make ~dir ~inherit_from ~scope ~config_stanza ~profile ~expander
           else
             Disabled)
   in
+  let js_of_ocaml =
+    inherited
+      ~field:(fun t -> js_of_ocaml t)
+      ~root:Js_of_ocaml.Env.(map ~f:Action_builder.return (default ~profile))
+      (fun (jsoo : _ Action_builder.t Js_of_ocaml.Env.t) ->
+        let local = config.js_of_ocaml in
+        let+ expander = Memo.Lazy.force expander in
+        let expander = Expander.set_dir expander ~dir in
+        let pick ~first ~second =
+          match first with
+          | None -> second
+          | Some _ as x -> x
+        in
+        { Js_of_ocaml.Env.compilation_mode =
+            pick ~first:local.compilation_mode ~second:jsoo.compilation_mode
+        ; runtest_alias =
+            pick ~first:local.runtest_alias ~second:jsoo.runtest_alias
+        ; flags =
+            Js_of_ocaml.Flags.make ~spec:local.flags ~default:jsoo.flags
+              ~eval:(Expander.expand_and_eval_set expander)
+        })
+  in
   let foreign_flags lang =
     let field t =
       Memo.Build.return (Foreign_language.Dict.get t.foreign_flags lang)
@@ -187,6 +212,7 @@ let make ~dir ~inherit_from ~scope ~config_stanza ~profile ~expander
   ; bin_artifacts
   ; local_binaries
   ; inline_tests
+  ; js_of_ocaml
   ; menhir_flags
   ; odoc
   ; coq

src/dune_rules/env_node.mli

@@ -38,6 +38,9 @@ val ocaml_flags : t -> Ocaml_flags.t Memo.Build.t
 
 val inline_tests : t -> Dune_env.Stanza.Inline_tests.t Memo.Build.t
 
+val js_of_ocaml :
+  t -> string list Action_builder.t Js_of_ocaml.Env.t Memo.Build.t
+
 val foreign_flags : t -> string list Action_builder.t Foreign_language.Dict.t
 
 val local_binaries : t -> File_binding.Expanded.t list Memo.Build.t