diff --git a/Makefile b/Makefile index 5e9ea8a44c66f..9ca3298573ff2 100644 --- a/Makefile +++ b/Makefile @@ -184,7 +184,7 @@ $(build_depsbindir)/stringreplace: $(JULIAHOME)/contrib/stringreplace.c | $(buil @$(call PRINT_CC, $(HOSTCC) -o $(build_depsbindir)/stringreplace $(JULIAHOME)/contrib/stringreplace.c) julia-base-cache: julia-sysimg-$(JULIA_BUILD_MODE) | $(DIRS) $(build_datarootdir)/julia - @JULIA_BINDIR=$(call cygpath_w,$(build_bindir)) WINEPATH="$(call cygpath_w,$(build_bindir));$$WINEPATH" \ + @JULIA_BINDIR=$(call cygpath_w,$(build_bindir)) JULIA_MINIMAL_CLIENT=1 WINEPATH="$(call cygpath_w,$(build_bindir));$$WINEPATH" \ $(call spawn, $(JULIA_EXECUTABLE) --startup-file=no $(call cygpath_w,$(JULIAHOME)/etc/write_base_cache.jl) \ $(call cygpath_w,$(build_datarootdir)/julia/base.cache)) diff --git a/base/client.jl b/base/client.jl index 0b11a330cf179..b8590861d7e80 100644 --- a/base/client.jl +++ b/base/client.jl @@ -227,13 +227,12 @@ end incomplete_tag(exc::Meta.ParseError) = incomplete_tag(exc.detail) cmd_suppresses_program(cmd) = cmd in ('e', 'E') + +# Minimal exec_options for precompilation function exec_options(opts) quiet = (opts.quiet != 0) - startup = (opts.startupfile != 2) - history_file = (opts.historyfile != 0) - color_set = (opts.color != 0) # --color!=auto - global have_color = color_set ? (opts.color == 1) : nothing # --color=on - global is_interactive = (opts.isinteractive != 0) + global have_color = false + global is_interactive = false # pre-process command line argument list arg_is_program = !isempty(ARGS) @@ -245,13 +244,6 @@ function exec_options(opts) repl = false elseif cmd == 'L' # nothing - elseif cmd == 'B' # --bug-report - # If we're doing a bug report, don't load anything else. We will - # spawn a child in which to execute these options. - let InteractiveUtils = load_InteractiveUtils() - InteractiveUtils.report_bug(arg) - end - return nothing else @warn "Unexpected command -$cmd'$arg'" end @@ -260,30 +252,6 @@ function exec_options(opts) # remove filename from ARGS global PROGRAM_FILE = arg_is_program ? popfirst!(ARGS) : "" - # Load Distributed module only if any of the Distributed options have been specified. - distributed_mode = (opts.worker == 1) || (opts.nprocs > 0) || (opts.machine_file != C_NULL) - if distributed_mode - let Distributed = require(PkgId(UUID((0x8ba89e20_285c_5b6f, 0x9357_94700520ee1b)), "Distributed")) - Core.eval(Main, :(const Distributed = $Distributed)) - Core.eval(Main, :(using .Distributed)) - end - - invokelatest(Main.Distributed.process_opts, opts) - end - - interactiveinput = (repl || is_interactive::Bool) && isa(stdin, TTY) - is_interactive::Bool |= interactiveinput - - # load ~/.julia/config/startup.jl file - if startup - try - load_julia_startup() - catch - invokelatest(display_error, scrub_repl_backtrace(current_exceptions())) - !(repl || is_interactive::Bool) && exit(1) - end - end - # process cmds list for (cmd, arg) in cmds if cmd == 'e' @@ -292,24 +260,13 @@ function exec_options(opts) invokelatest(show, Core.eval(Main, parse_input_line(arg))) println() elseif cmd == 'L' - # load file immediately on all processors - if !distributed_mode - include(Main, arg) - else - # TODO: Move this logic to Distributed and use a callback - @sync for p in invokelatest(Main.procs) - @async invokelatest(Main.remotecall_wait, include, p, Main, arg) - end - end + include(Main, arg) end end # load file if arg_is_program - # program - if !is_interactive::Bool - exit_on_sigint(true) - end + exit_on_sigint(true) try if PROGRAM_FILE == "-" include_string(Main, read(stdin, String), "stdin") @@ -318,52 +275,12 @@ function exec_options(opts) end catch invokelatest(display_error, scrub_repl_backtrace(current_exceptions())) - if !is_interactive::Bool - exit(1) - end + exit(1) end end - if repl || is_interactive::Bool - b = opts.banner - auto = b == -1 - banner = b == 0 || (auto && !interactiveinput) ? :no : - b == 1 || (auto && interactiveinput) ? :yes : - :short # b == 2 - run_main_repl(interactiveinput, quiet, banner, history_file, color_set) - end nothing end -function _global_julia_startup_file() - # If the user built us with a specific Base.SYSCONFDIR, check that location first for a startup.jl file - # If it is not found, then continue on to the relative path based on Sys.BINDIR - BINDIR = Sys.BINDIR - SYSCONFDIR = Base.SYSCONFDIR - if !isempty(SYSCONFDIR) - p1 = abspath(BINDIR, SYSCONFDIR, "julia", "startup.jl") - isfile(p1) && return p1 - end - p2 = abspath(BINDIR, "..", "etc", "julia", "startup.jl") - isfile(p2) && return p2 - return nothing -end - -function _local_julia_startup_file() - if !isempty(DEPOT_PATH) - path = abspath(DEPOT_PATH[1], "config", "startup.jl") - isfile(path) && return path - end - return nothing -end - -function load_julia_startup() - global_file = _global_julia_startup_file() - (global_file !== nothing) && include(Main, global_file) - local_file = _local_julia_startup_file() - (local_file !== nothing) && include(Main, local_file) - return nothing -end - const repl_hooks = [] """ @@ -388,96 +305,9 @@ function __atreplinit(repl) end _atreplinit(repl) = invokelatest(__atreplinit, repl) -function load_InteractiveUtils(mod::Module=Main) - # load interactive-only libraries - if !isdefined(mod, :InteractiveUtils) - try - let InteractiveUtils = require(PkgId(UUID(0xb77e0a4c_d291_57a0_90e8_8db25a27a240), "InteractiveUtils")) - Core.eval(mod, :(const InteractiveUtils = $InteractiveUtils)) - Core.eval(mod, :(using .InteractiveUtils)) - return InteractiveUtils - end - catch ex - @warn "Failed to import InteractiveUtils into module $mod" exception=(ex, catch_backtrace()) - end - return nothing - end - return getfield(mod, :InteractiveUtils) -end - +# set by Client.jl global active_repl -# run the requested sort of evaluation loop on stdio -function run_main_repl(interactive::Bool, quiet::Bool, banner::Symbol, history_file::Bool, color_set::Bool) - load_InteractiveUtils() - - if interactive && isassigned(REPL_MODULE_REF) - invokelatest(REPL_MODULE_REF[]) do REPL - term_env = get(ENV, "TERM", @static Sys.iswindows() ? "" : "dumb") - global current_terminfo = load_terminfo(term_env) - term = REPL.Terminals.TTYTerminal(term_env, stdin, stdout, stderr) - banner == :no || Base.banner(term, short=banner==:short) - if term.term_type == "dumb" - repl = REPL.BasicREPL(term) - quiet || @warn "Terminal not fully functional" - else - repl = REPL.LineEditREPL(term, get(stdout, :color, false), true) - repl.history_file = history_file - end - global active_repl = repl - # Make sure any displays pushed in .julia/config/startup.jl ends up above the - # REPLDisplay - pushdisplay(REPL.REPLDisplay(repl)) - _atreplinit(repl) - REPL.run_repl(repl, backend->(global active_repl_backend = backend)) - end - else - # otherwise provide a simple fallback - if interactive && !quiet - @warn "REPL provider not available: using basic fallback" - end - banner == :no || Base.banner(short=banner==:short) - let input = stdin - if isa(input, File) || isa(input, IOStream) - # for files, we can slurp in the whole thing at once - ex = parse_input_line(read(input, String)) - if Meta.isexpr(ex, :toplevel) - # if we get back a list of statements, eval them sequentially - # as if we had parsed them sequentially - for stmt in ex.args - eval_user_input(stderr, stmt, true) - end - body = ex.args - else - eval_user_input(stderr, ex, true) - end - else - while isopen(input) || !eof(input) - if interactive - print("julia> ") - flush(stdout) - end - try - line = "" - ex = nothing - while !eof(input) - line *= readline(input, keep=true) - ex = parse_input_line(line) - if !(isa(ex, Expr) && ex.head === :incomplete) - break - end - end - eval_user_input(stderr, ex, true) - catch err - isa(err, InterruptException) ? print("\n\n") : rethrow() - end - end - end - end - end - nothing -end - # MainInclude exists to hide Main.include and eval from `names(Main)`. baremodule MainInclude using ..Base @@ -549,7 +379,14 @@ function _start() # clear any postoutput hooks that were saved in the sysimage empty!(Base.postoutput_hooks) try - exec_options(JLOptions()) + if Base.generating_output() || parse(Bool, get(ENV, "JULIA_MINIMAL_CLIENT", "0")) + exec_options(JLOptions()) + else + let Client = require(PkgId(UUID((0x22d9f1f7_1f08_4fdd, 0xa466_30825a29bd37)), "Client")) + Core.eval(Main, :(const Client = $Client)) + invokelatest(Client.exec_options, JLOptions()) + end + end catch invokelatest(display_error, scrub_repl_backtrace(current_exceptions())) exit(1) diff --git a/base/sysimg.jl b/base/sysimg.jl index 04de951dc0199..3757a89212351 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -31,46 +31,14 @@ let # Run with the `--exclude-jlls` option to filter out all JLL packages stdlibs = [ # No dependencies - :ArgTools, - :Artifacts, - :Base64, - :CRC32c, - :FileWatching, - :Libdl, - :Logging, - :Mmap, - :NetworkOptions, - :SHA, - :Serialization, - :Sockets, - :Unicode, + :FileWatching, # used by loading.jl -- implicit assumption that init runs + :Libdl, # Transitive through LinAlg + :Artifacts, # Transitive through LinAlg + :SHA, # transitive through Random # 1-depth packages - :LinearAlgebra, - :Markdown, - :Printf, - :Random, - :Tar, - - # 2-depth packages - :Dates, - :Future, - :InteractiveUtils, - :LibGit2, - :UUIDs, - - # 3-depth packages - :REPL, - :TOML, - - # 4-depth packages - :LibCURL, - - # 5-depth packages - :Downloads, - - # 6-depth packages - :Pkg, + :LinearAlgebra, # Commits type-piracy and GEMM + :Random, # Can't be removed due to rand being exported by Base ] # PackageCompiler can filter out stdlibs so it can be empty maxlen = maximum(textwidth.(string.(stdlibs)); init=0) diff --git a/contrib/generate_precompile.jl b/contrib/generate_precompile.jl index fea4ca6bc1fe3..cfdaf3c79d9ca 100644 --- a/contrib/generate_precompile.jl +++ b/contrib/generate_precompile.jl @@ -33,6 +33,10 @@ UP_ARROW = "\e[A" DOWN_ARROW = "\e[B" hardcoded_precompile_statements = """ +# loading.jl +precompile(Base.__require_prelocked, (Base.PkgId, Nothing)) +precompile(Base._require, (Base.PkgId, Nothing)) + # used by Revise.jl precompile(Tuple{typeof(Base.parse_cache_header), String}) precompile(Base.read_dependency_src, (String, String)) @@ -127,8 +131,7 @@ precompile_script = """ julia_exepath() = joinpath(Sys.BINDIR, Base.julia_exename()) -have_repl = haskey(Base.loaded_modules, - Base.PkgId(Base.UUID("3fa0cd96-eef1-5676-8a61-b3b8758bbffb"), "REPL")) +have_repl = false if have_repl hardcoded_precompile_statements *= """ precompile(Tuple{typeof(getproperty), REPL.REPLBackend, Symbol}) @@ -153,9 +156,7 @@ if Artifacts !== nothing """ end -Pkg = get(Base.loaded_modules, - Base.PkgId(Base.UUID("44cfe95a-1eb2-52ea-b672-e2afdf69b78f"), "Pkg"), - nothing) +Pkg = nothing if Pkg !== nothing # TODO: Split Pkg precompile script into REPL and script part @@ -182,9 +183,7 @@ if Libdl !== nothing """ end -InteractiveUtils = get(Base.loaded_modules, - Base.PkgId(Base.UUID("b77e0a4c-d291-57a0-90e8-8db25a27a240"), "InteractiveUtils"), - nothing) +InteractiveUtils = nothing if InteractiveUtils !== nothing repl_script *= """ @time_imports using Random @@ -240,7 +239,8 @@ procenv = Dict{String,Any}( "JULIA_PROJECT" => nothing, # remove from environment "JULIA_LOAD_PATH" => "@stdlib", "JULIA_DEPOT_PATH" => Sys.iswindows() ? ";" : ":", - "TERM" => "") + "TERM" => "", + "JULIA_MINIMAL_CLIENT" => "true") generate_precompile_statements() = try # Make sure `ansi_enablecursor` is printed start_time = time_ns() diff --git a/pkgimage.mk b/pkgimage.mk index a721dc73d10d7..2bbb3d52b0eeb 100644 --- a/pkgimage.mk +++ b/pkgimage.mk @@ -11,6 +11,8 @@ export JULIA_LOAD_PATH := @stdlib unexport JULIA_PROJECT := unexport JULIA_BINDIR := +export JULIA_MINIMAL_CLIENT := true + default: release release: all-release debug: all-debug @@ -22,7 +24,7 @@ $(JULIA_DEPOT_PATH): print-depot-path: @$(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) --startup-file=no -e '@show Base.DEPOT_PATH') -STDLIBS := ArgTools Artifacts Base64 CRC32c FileWatching Libdl NetworkOptions SHA Serialization \ +STDLIBS := ArgTools Artifacts Base64 Client CRC32c FileWatching Libdl NetworkOptions SHA Serialization \ GMP_jll LLVMLibUnwind_jll LibUV_jll LibUnwind_jll MbedTLS_jll OpenLibm_jll PCRE2_jll \ Zlib_jll dSFMT_jll libLLVM_jll libblastrampoline_jll OpenBLAS_jll Printf Random Tar \ LibSSH2_jll MPFR_jll LinearAlgebra Dates Distributed Future LibGit2 Profile SparseArrays UUIDs \ @@ -63,27 +65,28 @@ endef # no dependencies $(eval $(call pkgimg_builder,MozillaCACerts_jll,)) -$(eval $(call sysimg_builder,ArgTools,)) +$(eval $(call pkgimg_builder,ArgTools,)) $(eval $(call sysimg_builder,Artifacts,)) -$(eval $(call sysimg_builder,Base64,)) -$(eval $(call sysimg_builder,CRC32c,)) +$(eval $(call pkgimg_builder,Base64,)) +$(eval $(call pkgimg_builder,CRC32c,)) $(eval $(call sysimg_builder,FileWatching,)) $(eval $(call sysimg_builder,Libdl,)) -$(eval $(call sysimg_builder,Logging,)) -$(eval $(call sysimg_builder,Mmap,)) -$(eval $(call sysimg_builder,NetworkOptions,)) +$(eval $(call pkgimg_builder,Logging,)) +$(eval $(call pkgimg_builder,Mmap,)) +$(eval $(call pkgimg_builder,NetworkOptions,)) $(eval $(call sysimg_builder,SHA,)) -$(eval $(call sysimg_builder,Serialization,)) -$(eval $(call sysimg_builder,Sockets,)) -$(eval $(call sysimg_builder,Unicode,)) +$(eval $(call pkgimg_builder,Serialization,)) +$(eval $(call pkgimg_builder,Sockets,)) +$(eval $(call pkgimg_builder,Unicode,)) $(eval $(call pkgimg_builder,Profile,)) +$(eval $(call pkgimg_builder,Client,)) # 1-depth packages $(eval $(call pkgimg_builder,GMP_jll,Artifacts Libdl)) $(eval $(call pkgimg_builder,LLVMLibUnwind_jll,Artifacts Libdl)) $(eval $(call pkgimg_builder,LibUV_jll,Artifacts Libdl)) $(eval $(call pkgimg_builder,LibUnwind_jll,Artifacts Libdl)) -$(eval $(call sysimg_builder,MbedTLS_jll,Artifacts Libdl)) +$(eval $(call pkgimg_builder,MbedTLS_jll,Artifacts Libdl)) $(eval $(call pkgimg_builder,nghttp2_jll,Artifacts Libdl)) $(eval $(call pkgimg_builder,OpenLibm_jll,Artifacts Libdl)) $(eval $(call pkgimg_builder,PCRE2_jll,Artifacts Libdl)) @@ -91,41 +94,44 @@ $(eval $(call pkgimg_builder,Zlib_jll,Artifacts Libdl)) $(eval $(call pkgimg_builder,dSFMT_jll,Artifacts Libdl)) $(eval $(call pkgimg_builder,libLLVM_jll,Artifacts Libdl)) $(eval $(call sysimg_builder,libblastrampoline_jll,Artifacts Libdl)) +$(eval $(call pkgimg_builder,p7zip_jll,Artifacts Libdl)) $(eval $(call sysimg_builder,OpenBLAS_jll,Artifacts Libdl)) -$(eval $(call sysimg_builder,Markdown,Base64)) -$(eval $(call sysimg_builder,Printf,Unicode)) +$(eval $(call pkgimg_builder,Markdown,Base64)) +$(eval $(call pkgimg_builder,Printf,Unicode)) $(eval $(call sysimg_builder,Random,SHA)) -$(eval $(call sysimg_builder,Tar,ArgTools,SHA)) +$(eval $(call pkgimg_builder,Tar,ArgTools,SHA)) $(eval $(call pkgimg_builder,DelimitedFiles,Mmap)) # 2-depth packages $(eval $(call pkgimg_builder,LLD_jll,Zlib_jll libLLVM_jll Artifacts Libdl)) -$(eval $(call sysimg_builder,LibSSH2_jll,Artifacts Libdl MbedTLS_jll)) +$(eval $(call pkgimg_builder,LibSSH2_jll,Artifacts Libdl MbedTLS_jll)) $(eval $(call pkgimg_builder,MPFR_jll,Artifacts Libdl GMP_jll)) $(eval $(call sysimg_builder,LinearAlgebra,Libdl libblastrampoline_jll OpenBLAS_jll)) -$(eval $(call sysimg_builder,Dates,Printf)) +$(eval $(call pkgimg_builder,Dates,Printf)) $(eval $(call pkgimg_builder,Distributed,Random Serialization Sockets)) -$(eval $(call sysimg_builder,Future,Random)) -$(eval $(call sysimg_builder,InteractiveUtils,Markdown)) -$(eval $(call sysimg_builder,UUIDs,Random SHA)) +$(eval $(call pkgimg_builder,Future,Random)) +$(eval $(call pkgimg_builder,UUIDs,Random SHA)) +$(eval $(call pkgimg_builder,InteractiveUtils,Markdown)) # 3-depth packages -$(eval $(call sysimg_builder,LibGit2_jll,MbedTLS_jll LibSSH2_jll Artifacts Libdl)) +$(eval $(call pkgimg_builder,LibGit2_jll,MbedTLS_jll LibSSH2_jll Artifacts Libdl)) $(eval $(call pkgimg_builder,LibCURL_jll,LibSSH2_jll nghttp2_jll MbedTLS_jll Zlib_jll Artifacts Libdl)) -$(eval $(call sysimg_builder,REPL,InteractiveUtils Markdown Sockets Unicode)) +$(eval $(call pkgimg_builder,REPL,InteractiveUtils Markdown Sockets Unicode Client)) $(eval $(call pkgimg_builder,SharedArrays,Distributed Mmap Random Serialization)) -$(eval $(call sysimg_builder,TOML,Dates)) +$(eval $(call pkgimg_builder,TOML,Dates)) $(eval $(call pkgimg_builder,Test,Logging Random Serialization InteractiveUtils)) # 4-depth packages -$(eval $(call sysimg_builder,LibGit2,LibGit2_jll NetworkOptions Printf SHA Base64)) -$(eval $(call sysimg_builder,LibCURL,LibCURL_jll MozillaCACerts_jll)) +$(eval $(call pkgimg_builder,LibGit2,LibGit2_jll NetworkOptions Printf SHA Base64)) +$(eval $(call pkgimg_builder,LibCURL,LibCURL_jll MozillaCACerts_jll)) # 5-depth packages -$(eval $(call sysimg_builder,Downloads,ArgTools FileWatching LibCURL NetworkOptions)) +$(eval $(call pkgimg_builder,Downloads,ArgTools FileWatching LibCURL NetworkOptions)) # 6-depth packages -$(eval $(call sysimg_builder,Pkg,Dates LibGit2 Libdl Logging Printf Random SHA UUIDs)) # Markdown REPL +$(eval $(call pkgimg_builder,Pkg, Artifacts Dates Downloads FileWatching LibGit2 Libdl\ + Logging Markdown Printf REPL Random SHA Serialization\ + TOML Tar UUIDs p7zip_jll)) # 7-depth packages $(eval $(call pkgimg_builder,LazyArtifacts,Artifacts Pkg)) diff --git a/stdlib/Client/Project.toml b/stdlib/Client/Project.toml new file mode 100644 index 0000000000000..e2ba0fe6757e8 --- /dev/null +++ b/stdlib/Client/Project.toml @@ -0,0 +1,3 @@ +name = "Client" +uuid = "22d9f1f7-1f08-4fdd-a466-30825a29bd37" +version = "1.11.0" diff --git a/stdlib/Client/src/Client.jl b/stdlib/Client/src/Client.jl new file mode 100644 index 0000000000000..fd89524c266fe --- /dev/null +++ b/stdlib/Client/src/Client.jl @@ -0,0 +1,244 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +module Client + +import Base: PkgId, UUID +using Base.Filesystem + +md_suppresses_program(cmd) = cmd in ('e', 'E') +function exec_options(opts) + quiet = (opts.quiet != 0) + startup = (opts.startupfile != 2) + history_file = (opts.historyfile != 0) + color_set = (opts.color != 0) # --color!=auto + Base.have_color = color_set ? (opts.color == 1) : nothing # --color=on + Base.is_interactive = (opts.isinteractive != 0) + + # pre-process command line argument list + arg_is_program = !isempty(ARGS) + repl = !arg_is_program + cmds = Base.unsafe_load_commands(opts.commands) + for (cmd, arg) in cmds + if Base.cmd_suppresses_program(cmd) + arg_is_program = false + repl = false + elseif cmd == 'L' + # nothing + elseif cmd == 'B' # --bug-report + # If we're doing a bug report, don't load anything else. We will + # spawn a child in which to execute these options. + let InteractiveUtils = load_InteractiveUtils() + InteractiveUtils.report_bug(arg) + end + return nothing + else + @warn "Unexpected command -$cmd'$arg'" + end + end + + # remove filename from ARGS + global PROGRAM_FILE = arg_is_program ? popfirst!(ARGS) : "" + + # Load Distributed module only if any of the Distributed options have been specified. + distributed_mode = (opts.worker == 1) || (opts.nprocs > 0) || (opts.machine_file != C_NULL) + if distributed_mode + let Distributed = require(PkgId(UUID((0x8ba89e20_285c_5b6f, 0x9357_94700520ee1b)), "Distributed")) + Core.eval(Main, :(const Distributed = $Distributed)) + Core.eval(Main, :(using .Distributed)) + end + + invokelatest(Main.Distributed.process_opts, opts) + end + + interactiveinput = (repl || Base.is_interactive::Bool) && isa(stdin, Base.TTY) + Base.is_interactive::Bool |= interactiveinput + + # load ~/.julia/config/startup.jl file + if startup + try + load_julia_startup() + catch + invokelatest(Base.display_error, Base.scrub_repl_backtrace(current_exceptions())) + !(repl || Base.is_interactive::Bool) && exit(1) + end + end + + # process cmds list + for (cmd, arg) in cmds + if cmd == 'e' + Core.eval(Main, Base.parse_input_line(arg)) + elseif cmd == 'E' + invokelatest(show, Core.eval(Main, Base.parse_input_line(arg))) + println() + elseif cmd == 'L' + # load file immediately on all processors + if !distributed_mode + include(Main, arg) + else + # TODO: Move this logic to Distributed and use a callback + @sync for p in invokelatest(Main.procs) + @async invokelatest(Main.remotecall_wait, include, p, Main, arg) + end + end + end + end + + # load file + if arg_is_program + # program + if !Base.is_interactive::Bool + Base.exit_on_sigint(true) + end + try + if PROGRAM_FILE == "-" + Base.include_string(Main, read(stdin, String), "stdin") + else + Base.include(Main, PROGRAM_FILE) + end + catch + invokelatest(Base.display_error, Base.scrub_repl_backtrace(current_exceptions())) + if !Base.is_interactive::Bool + exit(1) + end + end + end + if repl || Base.is_interactive::Bool + b = opts.banner + auto = b == -1 + banner = b == 0 || (auto && !interactiveinput) ? :no : + b == 1 || (auto && interactiveinput) ? :yes : + :short # b == 2 + run_main_repl(interactiveinput, quiet, banner, history_file, color_set) + end + nothing +end + +function _global_julia_startup_file() + # If the user built us with a specific Base.SYSCONFDIR, check that location first for a startup.jl file + # If it is not found, then continue on to the relative path based on Sys.BINDIR + BINDIR = Sys.BINDIR + SYSCONFDIR = Base.SYSCONFDIR + if !isempty(SYSCONFDIR) + p1 = abspath(BINDIR, SYSCONFDIR, "julia", "startup.jl") + isfile(p1) && return p1 + end + p2 = abspath(BINDIR, "..", "etc", "julia", "startup.jl") + isfile(p2) && return p2 + return nothing +end + +function _local_julia_startup_file() + if !isempty(DEPOT_PATH) + path = abspath(DEPOT_PATH[1], "config", "startup.jl") + isfile(path) && return path + end + return nothing +end + +function load_julia_startup() + global_file = _global_julia_startup_file() + (global_file !== nothing) && Base.include(Main, global_file) + local_file = _local_julia_startup_file() + (local_file !== nothing) && Base.include(Main, local_file) + return nothing +end + +function load_InteractiveUtils(mod::Module=Main) + # load interactive-only libraries + if !isdefined(mod, :InteractiveUtils) + try + let InteractiveUtils = Base.require(PkgId(UUID(0xb77e0a4c_d291_57a0_90e8_8db25a27a240), "InteractiveUtils")) + Core.eval(mod, :(const InteractiveUtils = $InteractiveUtils)) + Core.eval(mod, :(using .InteractiveUtils)) + return InteractiveUtils + end + catch ex + @warn "Failed to import InteractiveUtils into module $mod" exception=(ex, catch_backtrace()) + end + return nothing + end + return getfield(mod, :InteractiveUtils) +end + +# run the requested sort of evaluation loop on stdio +function run_main_repl(interactive::Bool, quiet::Bool, banner::Symbol, history_file::Bool, color_set::Bool) + load_InteractiveUtils() + + if interactive + let REPL = Base.require(PkgId(UUID(0x3fa0cd96_eef1_5676_8a61_b3b8758bbffb), "REPL")) + Core.eval(Main, :(const REPL = $REPL)) + Core.eval(Main, :(using .REPL)) + end + end + + if interactive && isassigned(Base.REPL_MODULE_REF) + invokelatest(Base.REPL_MODULE_REF[]) do REPL + term_env = get(ENV, "TERM", @static Sys.iswindows() ? "" : "dumb") + Base.current_terminfo = Base.load_terminfo(term_env) + term = REPL.Terminals.TTYTerminal(term_env, stdin, stdout, stderr) + banner == :no || Base.banner(term, short=banner==:short) + if term.term_type == "dumb" + repl = REPL.BasicREPL(term) + quiet || @warn "Terminal not fully functional" + else + repl = REPL.LineEditREPL(term, get(stdout, :color, false), true) + repl.history_file = history_file + end + Base.active_repl = repl + # Make sure any displays pushed in .julia/config/startup.jl ends up above the + # REPLDisplay + pushdisplay(REPL.REPLDisplay(repl)) + Base._atreplinit(repl) + REPL.run_repl(repl, backend->(Base.active_repl_backend = backend)) + end + else + # otherwise provide a simple fallback + # This hangs after Banner + if interactive && !quiet + @warn "REPL provider not available: using basic fallback" + end + banner == :no || Base.banner(short=banner==:short) + let input = stdin + if isa(input, File) || isa(input, IOStream) + # for files, we can slurp in the whole thing at once + ex = Base.parse_input_line(read(input, String)) + if Meta.isexpr(ex, :toplevel) + # if we get back a list of statements, eval them sequentially + # as if we had parsed them sequentially + for stmt in ex.args + Base.eval_user_input(stderr, stmt, true) + end + body = ex.args + else + Base.eval_user_input(stderr, ex, true) + end + else + while isopen(input) || !eof(input) + if interactive + print("julia> ") + flush(stdout) + end + try + line = "" + ex = nothing + while !eof(input) + line *= readline(input, keep=true) + ex = Base.parse_input_line(line) + if !(isa(ex, Expr) && ex.head === :incomplete) + break + end + end + Base.eval_user_input(stderr, ex, true) + catch err + isa(err, InterruptException) ? print("\n\n") : rethrow() + end + end + end + end + end + nothing +end + +precompile(Tuple{typeof(Client.exec_options), Base.JLOptions}) + +end # module Driver diff --git a/stdlib/Makefile b/stdlib/Makefile index e42061d593905..57e78b8577ad2 100644 --- a/stdlib/Makefile +++ b/stdlib/Makefile @@ -40,7 +40,7 @@ endef $(foreach jll,$(JLLS),$(eval $(call download-artifacts-toml,$(jll)))) -STDLIBS = Artifacts Base64 CRC32c Dates Distributed FileWatching \ +STDLIBS = Artifacts Base64 Client CRC32c Dates Distributed FileWatching \ Future InteractiveUtils LazyArtifacts Libdl LibGit2 LinearAlgebra Logging \ Markdown Mmap Printf Profile Random REPL Serialization \ SharedArrays Sockets Test TOML Unicode UUIDs \ diff --git a/stdlib/REPL/Project.toml b/stdlib/REPL/Project.toml index 4f77157da0146..1f1321cf824e3 100644 --- a/stdlib/REPL/Project.toml +++ b/stdlib/REPL/Project.toml @@ -2,14 +2,15 @@ name = "REPL" uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" [deps] +Client = "22d9f1f7-1f08-4fdd-a466-30825a29bd37" InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a" Sockets = "6462fe0b-24de-5631-8697-dd941f90decc" Unicode = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" [extras] -Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] test = ["Test", "Random"] diff --git a/stdlib/REPL/src/REPL.jl b/stdlib/REPL/src/REPL.jl index fe543ad2d512f..28ad9bb376607 100644 --- a/stdlib/REPL/src/REPL.jl +++ b/stdlib/REPL/src/REPL.jl @@ -1506,4 +1506,8 @@ end import .Numbered.numbered_prompt! +if Base.generating_output() + include("precompile.jl") +end + end # module diff --git a/stdlib/REPL/src/precompile.jl b/stdlib/REPL/src/precompile.jl new file mode 100644 index 0000000000000..05b09c01cfd7d --- /dev/null +++ b/stdlib/REPL/src/precompile.jl @@ -0,0 +1,202 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +module Precompile +# Can't use this during incremental: `@eval Module() begin`` + +import ..REPL + +Base.include(@__MODULE__, joinpath(Sys.BINDIR, "..", "share", "julia", "test", "testhelpers", "FakePTYs.jl")) +import .FakePTYs: open_fake_pty +using Base.Meta + +import Markdown +import Client + +## Debugging options +# Disable parallel precompiles generation by setting `false` +const PARALLEL_PRECOMPILATION = true + +# View the code sent to the repl by setting this to `stdout` +const debug_output = devnull # or stdout + +CTRL_C = '\x03' +CTRL_R = '\x12' +UP_ARROW = "\e[A" +DOWN_ARROW = "\e[B" + +repl_script = """ +2+2 +print("") +printstyled("a", "b") +display([1]) +display([1 2; 3 4]) +foo(x) = 1 +@time @eval foo(1) +; pwd +$CTRL_C +$CTRL_R$CTRL_C +? reinterpret +using Ra\t$CTRL_C +\\alpha\t$CTRL_C +\e[200~paste here ;)\e[201~"$CTRL_C +$UP_ARROW$DOWN_ARROW$CTRL_C +123\b\b\b$CTRL_C +\b\b$CTRL_C +f(x) = x03 +f(1,2) +[][1] +cd("complet_path\t\t$CTRL_C +""" + +julia_exepath() = joinpath(Sys.BINDIR, Base.julia_exename()) + +const JULIA_PROMPT = "julia> " +const PKG_PROMPT = "pkg> " +const SHELL_PROMPT = "shell> " +const HELP_PROMPT = "help?> " + +blackhole = Sys.isunix() ? "/dev/null" : "nul" +procenv = Dict{String,Any}( + "JULIA_HISTORY" => blackhole, + "JULIA_PROJECT" => nothing, # remove from environment + "JULIA_LOAD_PATH" => "@stdlib", + "JULIA_DEPOT_PATH" => Sys.iswindows() ? ";" : ":", + "TERM" => "", + "JULIA_MINIMAL_CLIENT" => "0") # Turn Client.jl on + +generate_precompile_statements() = try + # Extract the precompile statements from the precompile file + statements_step = Channel{String}(Inf) + + step = @async mktemp() do precompile_file, precompile_file_h + # Collect statements from running a REPL process and replaying our REPL script + touch(precompile_file) + pts, ptm = open_fake_pty() + cmdargs = `-e 'import REPL; REPL.Terminals.is_precompiling[] = true'` + p = run(addenv(addenv(```$(julia_exepath()) -O0 --trace-compile=$precompile_file + --cpu-target=native --startup-file=no --compiled-modules=existing --color=yes -i $cmdargs```, procenv), + "JULIA_PKG_PRECOMPILE_AUTO" => "0"), + pts, pts, pts; wait=false) + Base.close_stdio(pts) + # Prepare a background process to copy output from process until `pts` is closed + output_copy = Base.BufferStream() + tee = @async try + while !eof(ptm) + l = readavailable(ptm) + write(debug_output, l) + Sys.iswindows() && (sleep(0.1); yield(); yield()) # workaround hang - probably a libuv issue? + write(output_copy, l) + end + catch ex + if !(ex isa Base.IOError && ex.code == Base.UV_EIO) + rethrow() # ignore EIO on ptm after pts dies + end + finally + close(output_copy) + close(ptm) + end + Base.errormonitor(tee) + repl_inputter = @async begin + # wait for the definitive prompt before start writing to the TTY + readuntil(output_copy, JULIA_PROMPT) + sleep(0.1) + readavailable(output_copy) + # Input our script + precompile_lines = split(repl_script::String, '\n'; keepempty=false) + curr = 0 + for l in precompile_lines + sleep(0.1) + curr += 1 + # consume any other output + bytesavailable(output_copy) > 0 && readavailable(output_copy) + # push our input + write(debug_output, "\n#### inputting statement: ####\n$(repr(l))\n####\n") + write(ptm, l, "\n") + readuntil(output_copy, "\n") + # wait for the next prompt-like to appear + readuntil(output_copy, "\n") + strbuf = "" + while !eof(output_copy) + strbuf *= String(readavailable(output_copy)) + occursin(JULIA_PROMPT, strbuf) && break + occursin(PKG_PROMPT, strbuf) && break + occursin(SHELL_PROMPT, strbuf) && break + occursin(HELP_PROMPT, strbuf) && break + sleep(0.1) + end + end + write(ptm, "exit()\n") + wait(tee) + success(p) || Base.pipeline_error(p) + close(ptm) + write(debug_output, "\n#### FINISHED ####\n") + end + Base.errormonitor(repl_inputter) + + n_step = 0 + precompile_copy = Base.BufferStream() + buffer_reader = @async for statement in eachline(precompile_copy) + push!(statements_step, statement) + n_step += 1 + end + + open(precompile_file, "r") do io + while true + # We need to allways call eof(io) for bytesavailable(io) to work + eof(io) && istaskdone(repl_inputter) && eof(io) && break + if bytesavailable(io) == 0 + sleep(0.1) + continue + end + write(precompile_copy, readavailable(io)) + end + end + close(precompile_copy) + wait(buffer_reader) + close(statements_step) + return :ok + end + !PARALLEL_PRECOMPILATION && wait(step) + + # Make statements unique + statements = Set{String}() + # Execute the precompile statements + for statement in statements_step + # Main should be completely clean + occursin("Main.", statement) && continue + Base.in!(statement, statements) && continue + try + ps = Meta.parse(statement) + if !isexpr(ps, :call) + # these are typically comments + @debug "skipping statement because it does not parse as an expression" statement + delete!(statements, statement) + continue + end + popfirst!(ps.args) # precompile(...) + ps.head = :tuple + # println(ps) + ps = eval(ps) + if !precompile(ps...) + @warn "Failed to precompile expression" form=statement _module=nothing _file=nothing _line=0 + end + catch ex + # See #28808 + @warn "Failed to precompile expression" form=statement exception=ex _module=nothing _file=nothing _line=0 + end + end + + fetch(step) == :ok || throw("Collecting precompiles failed.") + return nothing +finally + GC.gc(true); GC.gc(false); # reduce memory footprint +end + +generate_precompile_statements() + +# As a last step in system image generation, +# remove some references to build time environment for a more reproducible build. +Base.Filesystem.temp_cleanup_purge(force=true) + +precompile(Tuple{typeof(getproperty), REPL.REPLBackend, Symbol}) +end # Precompile diff --git a/sysimage.mk b/sysimage.mk index 993ee9a990058..726e985b85065 100644 --- a/sysimage.mk +++ b/sysimage.mk @@ -53,6 +53,7 @@ COMPILER_SRCS += $(shell find $(JULIAHOME)/base/compiler -name \*.jl) # sort these to remove duplicates BASE_SRCS := $(sort $(shell find $(JULIAHOME)/base -name \*.jl -and -not -name sysimg.jl) \ $(shell find $(BUILDROOT)/base -name \*.jl -and -not -name sysimg.jl)) +# TODO: Avoid recompilation of base upon changes to pkgimage sources STDLIB_SRCS := $(JULIAHOME)/base/sysimg.jl $(shell find $(build_datarootdir)/julia/stdlib/$(VERSDIR)/*/src -name \*.jl) \ $(wildcard $(build_prefix)/manifest/$(VERSDIR)/*) RELBUILDROOT := $(call rel_path,$(JULIAHOME)/base,$(BUILDROOT)/base)/ # <-- make sure this always has a trailing slash