Merge pull request #46372 from JuliaLang/cjf/juliasyntax-stdlib

Enable JuliaSyntax.jl as an alternative Julia parser

Makefile

@@ -365,6 +365,10 @@ endif
 	# Remove various files which should not be installed
 	-rm -f $(DESTDIR)$(datarootdir)/julia/base/version_git.sh
 	-rm -f $(DESTDIR)$(datarootdir)/julia/test/Makefile
+	-rm -f $(DESTDIR)$(datarootdir)/julia/base/*/source-extracted
+	-rm -f $(DESTDIR)$(datarootdir)/julia/base/*/build-configured
+	-rm -f $(DESTDIR)$(datarootdir)/julia/base/*/build-compiled
+	-rm -f $(DESTDIR)$(datarootdir)/julia/base/*/build-checked
 	-rm -f $(DESTDIR)$(datarootdir)/julia/stdlib/$(VERSDIR)/*/source-extracted
 	-rm -f $(DESTDIR)$(datarootdir)/julia/stdlib/$(VERSDIR)/*/build-configured
 	-rm -f $(DESTDIR)$(datarootdir)/julia/stdlib/$(VERSDIR)/*/build-compiled

NEWS.md

@@ -4,6 +4,9 @@ Julia v1.10 Release Notes
 New language features
 ---------------------
 
+* JuliaSyntax.jl is now used as the default parser, providing better diagnostics and faster
+  parsing. Set environment variable `JULIA_USE_NEW_PARSER` to `0` to switch back to the old
+  parser if necessary (and if you find this necessary, please file an issue) ([#46372]).
 * `⥺` (U+297A, `\leftarrowsubset`) and `⥷` (U+2977, `\leftarrowless`)
   may now be used as binary operators with arrow precedence. ([#45962])
 

base/.gitignore

@@ -8,3 +8,4 @@
 /version_git.jl
 /version_git.jl.phony
 /userimg.jl
+/JuliaSyntax

base/Base.jl

@@ -489,6 +489,10 @@ a_method_to_overwrite_in_test() = inferencebarrier(1)
 include(mod::Module, _path::AbstractString) = _include(identity, mod, _path)
 include(mapexpr::Function, mod::Module, _path::AbstractString) = _include(mapexpr, mod, _path)
 
+# External libraries vendored into Base
+Core.println("JuliaSyntax/src/JuliaSyntax.jl")
+include(@__MODULE__, "JuliaSyntax/src/JuliaSyntax.jl")
+
 end_base_include = time_ns()
 
 const _sysimage_modules = PkgId[]
@@ -600,6 +604,9 @@ function __init__()
     _require_world_age[] = get_world_counter()
     # Prevent spawned Julia process from getting stuck waiting on Tracy to connect.
     delete!(ENV, "JULIA_WAIT_FOR_TRACY")
+    if get_bool_env("JULIA_USE_NEW_PARSER", true) === true
+        JuliaSyntax.enable_in_core!()
+    end
     nothing
 end
 

base/boot.jl

@@ -825,9 +825,11 @@ Integer(x::Union{Float16, Float32, Float64}) = Int(x)
 # `_parse` must return an `svec` containing an `Expr` and the new offset as an
 # `Int`.
 #
-# The internal jl_parse which will call into Core._parse if not `nothing`.
+# The internal jl_parse will call into Core._parse if not `nothing`.
 _parse = nothing
 
+_setparser!(parser) = setglobal!(Core, :_parse, parser)
+
 # support for deprecated uses of internal _apply function
 _apply(x...) = Core._apply_iterate(Main.Base.iterate, x...)
 

base/client.jl

@@ -202,10 +202,7 @@ parse_input_line(s::AbstractString) = parse_input_line(String(s))
 # detect the reason which caused an :incomplete expression
 # from the error message
 # NOTE: the error messages are defined in src/julia-parser.scm
-incomplete_tag(ex) = :none
-function incomplete_tag(ex::Expr)
-    Meta.isexpr(ex, :incomplete) || return :none
-    msg = ex.args[1]
+function fl_incomplete_tag(msg::AbstractString)
     occursin("string", msg) && return :string
     occursin("comment", msg) && return :comment
     occursin("requires end", msg) && return :block
@@ -214,6 +211,20 @@ function incomplete_tag(ex::Expr)
     return :other
 end
 
+incomplete_tag(ex) = :none
+function incomplete_tag(ex::Expr)
+    if ex.head !== :incomplete
+        return :none
+    elseif isempty(ex.args)
+        return :other
+    elseif ex.args[1] isa String
+        return fl_incomplete_tag(ex.args[1])
+    else
+        return incomplete_tag(ex.args[1])
+    end
+end
+incomplete_tag(exc::Meta.ParseError) = incomplete_tag(exc.detail)
+
 function exec_options(opts)
     quiet                 = (opts.quiet != 0)
     startup               = (opts.startupfile != 2)

base/compiler/compiler.jl

@@ -171,7 +171,7 @@ include("compiler/bootstrap.jl")
 ccall(:jl_set_typeinf_func, Cvoid, (Any,), typeinf_ext_toplevel)
 
 include("compiler/parsing.jl")
-Core.eval(Core, :(_parse = Compiler.fl_parse))
+Core._setparser!(fl_parse)
 
 end # baremodule Compiler
 ))

base/errorshow.jl

@@ -35,6 +35,13 @@ show_index(io::IO, x::LogicalIndex) = summary(io, x.mask)
 show_index(io::IO, x::OneTo) = print(io, "1:", x.stop)
 show_index(io::IO, x::Colon) = print(io, ':')
 
+function showerror(io::IO, ex::Meta.ParseError)
+    if isnothing(ex.detail)
+        print(io, "ParseError(", repr(ex.msg), ")")
+    else
+        showerror(io, ex.detail)
+    end
+end
 
 function showerror(io::IO, ex::BoundsError)
     print(io, "BoundsError")

base/meta.jl

@@ -187,8 +187,11 @@ expression.
 """
 struct ParseError <: Exception
     msg::String
+    detail::Any
 end
 
+ParseError(msg::AbstractString) = ParseError(msg, nothing)
+
 function _parse_string(text::AbstractString, filename::AbstractString,
                        lineno::Integer, index::Integer, options)
     if index < 1 || index > ncodeunits(text) + 1
@@ -233,7 +236,11 @@ function parse(str::AbstractString, pos::Integer; greedy::Bool=true, raise::Bool
                depwarn::Bool=true)
     ex, pos = _parse_string(str, "none", 1, pos, greedy ? :statement : :atom)
     if raise && isa(ex,Expr) && ex.head === :error
-        throw(ParseError(ex.args[1]))
+        err = ex.args[1]
+        if err isa String
+            err = ParseError(err) # For flisp parser
+        end
+        throw(err)
     end
     return ex, pos
 end
@@ -247,20 +254,22 @@ syntax errors will raise an error; otherwise, `parse` will return an expression
 raise an error upon evaluation.  If `depwarn` is `false`, deprecation warnings will be
 suppressed.
 
-```jldoctest
+```jldoctest; filter=r"(?<=Expr\\(:error).*|(?<=Expr\\(:incomplete).*"
 julia> Meta.parse("x = 3")
 :(x = 3)
 
-julia> Meta.parse("x = ")
-:($(Expr(:incomplete, "incomplete: premature end of input")))
-
 julia> Meta.parse("1.0.2")
-ERROR: Base.Meta.ParseError("invalid numeric constant \\\"1.0.\\\"")
-Stacktrace:
+ERROR: ParseError:
+# Error @ none:1:1
+1.0.2
+└──┘ ── invalid numeric constant
 [...]
 
 julia> Meta.parse("1.0.2"; raise = false)
-:($(Expr(:error, "invalid numeric constant \"1.0.\"")))
+:(\$(Expr(:error, "invalid numeric constant \"1.0.\"")))
+
+julia> Meta.parse("x = ")
+:(\$(Expr(:incomplete, "incomplete: premature end of input")))
 ```
 """
 function parse(str::AbstractString; raise::Bool=true, depwarn::Bool=true)

contrib/generate_precompile.jl

@@ -153,7 +153,6 @@ if Artifacts !== nothing
     """
 end
 
-
 Pkg = get(Base.loaded_modules,
           Base.PkgId(Base.UUID("44cfe95a-1eb2-52ea-b672-e2afdf69b78f"), "Pkg"),
           nothing)

deps/JuliaSyntax.mk

@@ -0,0 +1,16 @@
+$(eval $(call git-external,JuliaSyntax,JULIASYNTAX,,,$(BUILDDIR)))
+
+$(BUILDDIR)/$(JULIASYNTAX_SRC_DIR)/build-compiled: $(BUILDDIR)/$(JULIASYNTAX_SRC_DIR)/source-extracted
+	@# no build steps
+	echo 1 > $@
+
+$(eval $(call symlink_install,JuliaSyntax,$$(JULIASYNTAX_SRC_DIR),$$(JULIAHOME)/base))
+
+clean-JuliaSyntax:
+	-rm -f $(BUILDDIR)/$(JULIASYNTAX_SRC_DIR)/build-compiled
+get-JuliaSyntax: $(JULIASYNTAX_SRC_FILE)
+extract-JuliaSyntax: $(BUILDDIR)/$(JULIASYNTAX_SRC_DIR)/source-extracted
+configure-JuliaSyntax: extract-JuliaSyntax
+compile-JuliaSyntax: $(BUILDDIR)/$(JULIASYNTAX_SRC_DIR)/build-compiled
+fastcheck-JuliSyntax: check-JuliSyntax
+check-JuliSyntax: compile-JuliSyntax

deps/JuliaSyntax.version

@@ -0,0 +1,4 @@
+JULIASYNTAX_BRANCH = main
+JULIASYNTAX_SHA1 = ec51994833d78f8c5525bc1647f448dfadc370c1
+JULIASYNTAX_GIT_URL := https://github.com/JuliaLang/JuliaSyntax.jl.git
+JULIASYNTAX_TAR_URL = https://api.github.com/repos/JuliaLang/JuliaSyntax.jl/tarball/$1

deps/Makefile

@@ -36,7 +36,7 @@ BUILDDIR := $(BUILDDIR)$(MAYBE_HOST)
 # prevent installing libs into usr/lib64 on opensuse
 unexport CONFIG_SITE
 
-DEP_LIBS :=
+DEP_LIBS := JuliaSyntax
 
 ifeq ($(USE_SYSTEM_LIBBLASTRAMPOLINE), 0)
 DEP_LIBS += blastrampoline
@@ -188,7 +188,7 @@ DEP_LIBS_STAGED := $(DEP_LIBS)
 DEP_LIBS_STAGED_ALL := llvm llvm-tools clang llvmunwind unwind libuv pcre \
 	openlibm dsfmt blastrampoline openblas lapack gmp mpfr patchelf utf8proc \
 	objconv mbedtls libssh2 nghttp2 curl libgit2 libwhich zlib p7zip csl \
-	libsuitesparse lld libtracyclient ittapi
+	libsuitesparse lld libtracyclient ittapi JuliaSyntax
 DEP_LIBS_ALL := $(DEP_LIBS_STAGED_ALL)
 
 ifneq ($(USE_BINARYBUILDER_OPENBLAS),0)
@@ -248,4 +248,7 @@ include $(SRCDIR)/libwhich.mk
 include $(SRCDIR)/p7zip.mk
 include $(SRCDIR)/libtracyclient.mk
 
+# vendored Julia libs
+include $(SRCDIR)/JuliaSyntax.mk
+
 include $(SRCDIR)/tools/uninstallers.mk

deps/checksums/JuliaSyntax-ec51994833d78f8c5525bc1647f448dfadc370c1.tar.gz/md5

@@ -0,0 +1 @@
+b1d1ccb00e422eb8b70b2120d7083bf3

deps/checksums/JuliaSyntax-ec51994833d78f8c5525bc1647f448dfadc370c1.tar.gz/sha512

@@ -0,0 +1 @@
+e6df6dc2b5d2a5618da0d553eed793e1192147175d84d51f725c0ea8f7b6be92fbeb37de9abee2b2f548b0f0736f836ec7e3e20e93c12f77e1a2b2058bbfd6db

src/ast.c

@@ -1348,8 +1348,8 @@ jl_value_t *jl_parse(const char *text, size_t text_len, jl_value_t *filename,
     jl_svecset(args[1], 0, jl_box_uint8pointer((uint8_t*)text));
     jl_svecset(args[1], 1, jl_box_long(text_len));
     args[2] = filename;
-    args[3] = jl_box_ulong(lineno);
-    args[4] = jl_box_ulong(offset);
+    args[3] = jl_box_long(lineno);
+    args[4] = jl_box_long(offset);
     args[5] = options;
     jl_task_t *ct = jl_current_task;
     size_t last_age = ct->world_age;

src/toplevel.c

@@ -656,19 +656,28 @@ static void check_macro_rename(jl_sym_t *from, jl_sym_t *to, const char *keyword
         jl_errorf("cannot rename non-macro \"%s\" to macro \"%s\" in \"%s\"", n1, n2, keyword);
 }
 
-// Format msg and eval `throw(ErrorException(msg)))` in module `m`.
-// Used in `jl_toplevel_eval_flex` instead of `jl_errorf` so that the error
+// Eval `throw(ErrorException(msg)))` in module `m`.
+// Used in `jl_toplevel_eval_flex` instead of `jl_throw` so that the error
 // location in julia code gets into the backtrace.
-static void jl_eval_errorf(jl_module_t *m, const char* fmt, ...)
+static void jl_eval_throw(jl_module_t *m, jl_value_t *exc)
 {
     jl_value_t *throw_ex = (jl_value_t*)jl_exprn(jl_call_sym, 2);
     JL_GC_PUSH1(&throw_ex);
     jl_exprargset(throw_ex, 0, jl_builtin_throw);
+    jl_exprargset(throw_ex, 1, exc);
+    jl_toplevel_eval_flex(m, throw_ex, 0, 0);
+    JL_GC_POP();
+}
+
+// Format error message and call jl_eval
+static void jl_eval_errorf(jl_module_t *m, const char* fmt, ...)
+{
     va_list args;
     va_start(args, fmt);
-    jl_exprargset(throw_ex, 1, jl_vexceptionf(jl_errorexception_type, fmt, args));
+    jl_value_t *exc = jl_vexceptionf(jl_errorexception_type, fmt, args);
     va_end(args);
-    jl_toplevel_eval_flex(m, throw_ex, 0, 0);
+    JL_GC_PUSH1(&exc);
+    jl_eval_throw(m, exc);
     JL_GC_POP();
 }
 
@@ -875,7 +884,7 @@ jl_value_t *jl_toplevel_eval_flex(jl_module_t *JL_NONNULL m, jl_value_t *e, int
             jl_eval_errorf(m, "malformed \"%s\" expression", jl_symbol_name(head));
         if (jl_is_string(jl_exprarg(ex, 0)))
             jl_eval_errorf(m, "syntax: %s", jl_string_data(jl_exprarg(ex, 0)));
-        jl_throw(jl_exprarg(ex, 0));
+        jl_eval_throw(m, jl_exprarg(ex, 0));
     }
     else if (jl_is_symbol(ex)) {
         JL_GC_POP();

stdlib/REPL/src/REPLCompletions.jl

@@ -232,7 +232,10 @@ function complete_keyword(s::Union{String,SubString{String}})
     Completion[KeywordCompletion(kw) for kw in sorted_keywords[r]]
 end
 
-function complete_path(path::AbstractString, pos::Int; use_envpath=false, shell_escape=false)
+function complete_path(path::AbstractString, pos::Int;
+                       use_envpath=false, shell_escape=false,
+                       string_escape=false)
+    @assert !(shell_escape && string_escape)
     if Base.Sys.isunix() && occursin(r"^~(?:/|$)", path)
         # if the path is just "~", don't consider the expanded username as a prefix
         if path == "~"
@@ -259,9 +262,9 @@ function complete_path(path::AbstractString, pos::Int; use_envpath=false, shell_
     matches = Set{String}()
     for file in files
         if startswith(file, prefix)
-            id = try isdir(joinpath(dir, file)) catch; false end
-            # joinpath is not used because windows needs to complete with double-backslash
-            push!(matches, id ? file * (@static Sys.iswindows() ? "\\\\" : "/") : file)
+            p = joinpath(dir, file)
+            is_dir = try isdir(p) catch; false end
+            push!(matches, is_dir ? joinpath(file, "") : file)
         end
     end
 
@@ -307,8 +310,14 @@ function complete_path(path::AbstractString, pos::Int; use_envpath=false, shell_
         end
     end
 
-    matchList = Completion[PathCompletion(shell_escape ? replace(s, r"\s" => s"\\\0") : s) for s in matches]
-    startpos = pos - lastindex(prefix) + 1 - count(isequal(' '), prefix)
+    function do_escape(s)
+        return shell_escape ? replace(s, r"(\s|\\)" => s"\\\0") :
+               string_escape ? escape_string(s, ('\"','$')) :
+               s
+    end
+
+    matchList = Completion[PathCompletion(do_escape(s)) for s in matches]
+    startpos = pos - lastindex(do_escape(prefix)) + 1
     # The pos - lastindex(prefix) + 1 is correct due to `lastindex(prefix)-lastindex(prefix)==0`,
     # hence we need to add one to get the first index. This is also correct when considering
     # pos, because pos is the `lastindex` a larger string which `endswith(path)==true`.
@@ -767,7 +776,7 @@ end
 function close_path_completion(str, startpos, r, paths, pos)
     length(paths) == 1 || return false  # Only close if there's a single choice...
     _path = str[startpos:prevind(str, first(r))] * (paths[1]::PathCompletion).path
-    path = expanduser(replace(_path, r"\\ " => " "))
+    path = expanduser(unescape_string(replace(_path, "\\\$"=>"\$", "\\\""=>"\"")))
     # ...except if it's a directory...
     try
         isdir(path)
@@ -1039,23 +1048,44 @@ function completions(string::String, pos::Int, context_module::Module=Main, shif
         dotpos = something(findprev(isequal('.'), string, first(varrange)-1), 0)
         return complete_identifiers!(Completion[], ffunc, context_module, string,
             string[startpos:pos], pos, dotpos, startpos)
-    # otherwise...
-    elseif inc_tag in [:cmd, :string]
+    elseif inc_tag === :cmd
         m = match(r"[\t\n\r\"`><=*?|]| (?!\\)", reverse(partial))
         startpos = nextind(partial, reverseind(partial, m.offset))
         r = startpos:pos
 
+        # This expansion with "\\ "=>' ' replacement and shell_escape=true
+        # assumes the path isn't further quoted within the cmd backticks.
         expanded = complete_expanduser(replace(string[r], r"\\ " => " "), r)
         expanded[3] && return expanded  # If user expansion available, return it
 
-        paths, r, success = complete_path(replace(string[r], r"\\ " => " "), pos)
+        paths, r, success = complete_path(replace(string[r], r"\\ " => " "), pos,
+                                          shell_escape=true)
+
+        return sort!(paths, by=p->p.path), r, success
+    elseif inc_tag === :string
+        # Find first non-escaped quote
+        m = match(r"\"(?!\\)", reverse(partial))
+        startpos = nextind(partial, reverseind(partial, m.offset))
+        r = startpos:pos
+
+        expanded = complete_expanduser(string[r], r)
+        expanded[3] && return expanded  # If user expansion available, return it
 
-        if inc_tag === :string && close_path_completion(string, startpos, r, paths, pos)
-            paths[1] = PathCompletion((paths[1]::PathCompletion).path * "\"")
+        path_prefix = try
+            unescape_string(replace(string[r], "\\\$"=>"\$", "\\\""=>"\""))
+        catch
+            nothing
         end
+        if !isnothing(path_prefix)
+            paths, r, success = complete_path(path_prefix, pos, string_escape=true)
 
-        #Latex symbols can be completed for strings
-        (success || inc_tag === :cmd) && return sort!(paths, by=p->p.path), r, success
+            if close_path_completion(string, startpos, r, paths, pos)
+                paths[1] = PathCompletion((paths[1]::PathCompletion).path * "\"")
+            end
+
+            # Fallthrough allowed so that Latex symbols can be completed in strings
+            success && return sort!(paths, by=p->p.path), r, success
+        end
     end
 
     ok, ret = bslash_completions(string, pos)