diff --git a/compiler/rustc_codegen_cranelift/.vscode/settings.json b/compiler/rustc_codegen_cranelift/.vscode/settings.json index 04ab5085c196c..7618251acd5c2 100644 --- a/compiler/rustc_codegen_cranelift/.vscode/settings.json +++ b/compiler/rustc_codegen_cranelift/.vscode/settings.json @@ -1,6 +1,7 @@ { // source for rustc_* is not included in the rust-src component; disable the errors about this "rust-analyzer.diagnostics.disabled": ["unresolved-extern-crate"], + "rust-analyzer.assist.importMergeBehaviour": "last", "rust-analyzer.cargo.loadOutDirsFromCheck": true, "rust-analyzer.linkedProjects": [ "./Cargo.toml", diff --git a/compiler/rustc_codegen_cranelift/Cargo.lock b/compiler/rustc_codegen_cranelift/Cargo.lock index 67ed41e765231..0382835269d1f 100644 --- a/compiler/rustc_codegen_cranelift/Cargo.lock +++ b/compiler/rustc_codegen_cranelift/Cargo.lock @@ -50,7 +50,7 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "cranelift-bforest" version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8f7f8ee0b4c5007ace6de29b45505c360450b1bb" dependencies = [ "cranelift-entity", ] @@ -58,7 +58,7 @@ dependencies = [ [[package]] name = "cranelift-codegen" version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8f7f8ee0b4c5007ace6de29b45505c360450b1bb" dependencies = [ "byteorder", "cranelift-bforest", @@ -76,7 +76,7 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8f7f8ee0b4c5007ace6de29b45505c360450b1bb" dependencies = [ "cranelift-codegen-shared", "cranelift-entity", @@ -85,17 +85,17 @@ dependencies = [ [[package]] name = "cranelift-codegen-shared" version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8f7f8ee0b4c5007ace6de29b45505c360450b1bb" [[package]] name = "cranelift-entity" version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8f7f8ee0b4c5007ace6de29b45505c360450b1bb" [[package]] name = "cranelift-frontend" version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8f7f8ee0b4c5007ace6de29b45505c360450b1bb" dependencies = [ "cranelift-codegen", "log", @@ -103,10 +103,28 @@ dependencies = [ "target-lexicon", ] +[[package]] +name = "cranelift-jit" +version = "0.68.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8f7f8ee0b4c5007ace6de29b45505c360450b1bb" +dependencies = [ + "anyhow", + "cranelift-codegen", + "cranelift-entity", + "cranelift-module", + "cranelift-native", + "errno", + "libc", + "log", + "region", + "target-lexicon", + "winapi", +] + [[package]] name = "cranelift-module" version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8f7f8ee0b4c5007ace6de29b45505c360450b1bb" dependencies = [ "anyhow", "cranelift-codegen", @@ -118,7 +136,7 @@ dependencies = [ [[package]] name = "cranelift-native" version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8f7f8ee0b4c5007ace6de29b45505c360450b1bb" dependencies = [ "cranelift-codegen", "raw-cpuid", @@ -128,7 +146,7 @@ dependencies = [ [[package]] name = "cranelift-object" version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8f7f8ee0b4c5007ace6de29b45505c360450b1bb" dependencies = [ "anyhow", "cranelift-codegen", @@ -138,23 +156,6 @@ dependencies = [ "target-lexicon", ] -[[package]] -name = "cranelift-simplejit" -version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3" -dependencies = [ - "cranelift-codegen", - "cranelift-entity", - "cranelift-module", - "cranelift-native", - "errno", - "libc", - "log", - "region", - "target-lexicon", - "winapi", -] - [[package]] name = "crc32fast" version = "1.2.1" @@ -325,9 +326,9 @@ dependencies = [ "ar", "cranelift-codegen", "cranelift-frontend", + "cranelift-jit", "cranelift-module", "cranelift-object", - "cranelift-simplejit", "gimli", "indexmap", "libloading", diff --git a/compiler/rustc_codegen_cranelift/Cargo.toml b/compiler/rustc_codegen_cranelift/Cargo.toml index cbff06749d3e9..8e1933bb14e7c 100644 --- a/compiler/rustc_codegen_cranelift/Cargo.toml +++ b/compiler/rustc_codegen_cranelift/Cargo.toml @@ -12,7 +12,7 @@ crate-type = ["dylib"] cranelift-codegen = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main", features = ["unwind"] } cranelift-frontend = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main" } cranelift-module = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main" } -cranelift-simplejit = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main", optional = true } +cranelift-jit = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main", optional = true } cranelift-object = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main" } target-lexicon = "0.11.0" gimli = { version = "0.23.0", default-features = false, features = ["write"]} @@ -27,7 +27,7 @@ libloading = { version = "0.6.0", optional = true } #cranelift-codegen = { path = "../wasmtime/cranelift/codegen" } #cranelift-frontend = { path = "../wasmtime/cranelift/frontend" } #cranelift-module = { path = "../wasmtime/cranelift/module" } -#cranelift-simplejit = { path = "../wasmtime/cranelift/simplejit" } +#cranelift-jit = { path = "../wasmtime/cranelift/jit" } #cranelift-object = { path = "../wasmtime/cranelift/object" } #[patch.crates-io] @@ -35,7 +35,7 @@ libloading = { version = "0.6.0", optional = true } [features] default = ["jit", "inline_asm"] -jit = ["cranelift-simplejit", "libloading"] +jit = ["cranelift-jit", "libloading"] inline_asm = [] [profile.dev] diff --git a/compiler/rustc_codegen_cranelift/Readme.md b/compiler/rustc_codegen_cranelift/Readme.md index de54bf67f4a19..22d9e00923f00 100644 --- a/compiler/rustc_codegen_cranelift/Readme.md +++ b/compiler/rustc_codegen_cranelift/Readme.md @@ -2,7 +2,7 @@ > ⚠⚠⚠ Certain kinds of FFI don't work yet. ⚠⚠⚠ -The goal of this project is to create an alternative codegen backend for the rust compiler based on [Cranelift](https://github.com/bytecodealliance/wasmtime/blob/master/cranelift). +The goal of this project is to create an alternative codegen backend for the rust compiler based on [Cranelift](https://github.com/bytecodealliance/wasmtime/blob/main/cranelift). This has the potential to improve compilation times in debug mode. If your project doesn't use any of the things listed under "Not yet supported", it should work fine. If not please open an issue. @@ -68,7 +68,15 @@ $ $cg_clif_dir/build/cargo.sh jit or ```bash -$ $cg_clif_dir/build/bin/cg_clif --jit my_crate.rs +$ $cg_clif_dir/build/bin/cg_clif -Cllvm-args=mode=jit -Cprefer-dynamic my_crate.rs +``` + +There is also an experimental lazy jit mode. In this mode functions are only compiled once they are +first called. It currently does not work with multi-threaded programs. When a not yet compiled +function is called from another thread than the main thread, you will get an ICE. + +```bash +$ $cg_clif_dir/build/cargo.sh lazy-jit ``` ### Shell @@ -77,7 +85,7 @@ These are a few functions that allow you to easily run rust code from the shell ```bash function jit_naked() { - echo "$@" | $cg_clif_dir/build/bin/cg_clif - --jit + echo "$@" | $cg_clif_dir/build/bin/cg_clif - -Cllvm-args=mode=jit -Cprefer-dynamic } function jit() { diff --git a/compiler/rustc_codegen_cranelift/build_sysroot/Cargo.lock b/compiler/rustc_codegen_cranelift/build_sysroot/Cargo.lock index a2b8f449f00ff..990557694ead4 100644 --- a/compiler/rustc_codegen_cranelift/build_sysroot/Cargo.lock +++ b/compiler/rustc_codegen_cranelift/build_sysroot/Cargo.lock @@ -47,9 +47,9 @@ checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" [[package]] name = "cc" -version = "1.0.65" +version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95752358c8f7552394baf48cd82695b345628ad3f170d607de3ca03b8dacca15" +checksum = "4c0496836a84f8d0495758516b8621a622beb77c0fed418570e50764093ced48" [[package]] name = "cfg-if" @@ -141,9 +141,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.80" +version = "0.2.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d58d1b70b004888f764dfbf6a26a3b0342a1632d33968e4a179d8011c760614" +checksum = "1482821306169ec4d07f6aca392a4681f66c75c9918aa49641a2595db64053cb" dependencies = [ "rustc-std-workspace-core", ] diff --git a/compiler/rustc_codegen_cranelift/build_sysroot/Cargo.toml b/compiler/rustc_codegen_cranelift/build_sysroot/Cargo.toml index e562dedb5324b..3dbd28c286a24 100644 --- a/compiler/rustc_codegen_cranelift/build_sysroot/Cargo.toml +++ b/compiler/rustc_codegen_cranelift/build_sysroot/Cargo.toml @@ -5,13 +5,14 @@ version = "0.0.0" [dependencies] core = { path = "./sysroot_src/library/core" } -compiler_builtins = "0.1" alloc = { path = "./sysroot_src/library/alloc" } std = { path = "./sysroot_src/library/std", features = ["panic_unwind", "backtrace"] } test = { path = "./sysroot_src/library/test" } alloc_system = { path = "./alloc_system" } +compiler_builtins = { version = "=0.1.36", default-features = false } + [patch.crates-io] rustc-std-workspace-core = { path = "./sysroot_src/library/rustc-std-workspace-core" } rustc-std-workspace-alloc = { path = "./sysroot_src/library/rustc-std-workspace-alloc" } diff --git a/compiler/rustc_codegen_cranelift/example/std_example.rs b/compiler/rustc_codegen_cranelift/example/std_example.rs index b38e25328a4ee..015bbdfed4648 100644 --- a/compiler/rustc_codegen_cranelift/example/std_example.rs +++ b/compiler/rustc_codegen_cranelift/example/std_example.rs @@ -15,6 +15,8 @@ fn main() { let stderr = ::std::io::stderr(); let mut stderr = stderr.lock(); + // FIXME support lazy jit when multi threading + #[cfg(not(lazy_jit))] std::thread::spawn(move || { println!("Hello from another thread!"); }); diff --git a/compiler/rustc_codegen_cranelift/rust-toolchain b/compiler/rustc_codegen_cranelift/rust-toolchain index ed1e64f45db08..d6ad24bcf26dd 100644 --- a/compiler/rustc_codegen_cranelift/rust-toolchain +++ b/compiler/rustc_codegen_cranelift/rust-toolchain @@ -1 +1 @@ -nightly-2020-11-27 +nightly-2020-12-23 diff --git a/compiler/rustc_codegen_cranelift/scripts/cargo.sh b/compiler/rustc_codegen_cranelift/scripts/cargo.sh index dcd40acc02a53..a3d6d303057b8 100755 --- a/compiler/rustc_codegen_cranelift/scripts/cargo.sh +++ b/compiler/rustc_codegen_cranelift/scripts/cargo.sh @@ -10,7 +10,9 @@ cmd=$1 shift || true if [[ "$cmd" = "jit" ]]; then -cargo "+${TOOLCHAIN}" rustc "$@" -- --jit +cargo "+${TOOLCHAIN}" rustc "$@" -- -Cllvm-args=mode=jit -Cprefer-dynamic +elif [[ "$cmd" = "lazy-jit" ]]; then +cargo "+${TOOLCHAIN}" rustc "$@" -- -Cllvm-args=mode=jit-lazy -Cprefer-dynamic else cargo "+${TOOLCHAIN}" "$cmd" "$@" fi diff --git a/compiler/rustc_codegen_cranelift/scripts/filter_profile.rs b/compiler/rustc_codegen_cranelift/scripts/filter_profile.rs index 3327c10089d9b..15388926ec9ec 100755 --- a/compiler/rustc_codegen_cranelift/scripts/filter_profile.rs +++ b/compiler/rustc_codegen_cranelift/scripts/filter_profile.rs @@ -4,7 +4,7 @@ pushd $(dirname "$0")/../ source build/config.sh popd -PROFILE=$1 OUTPUT=$2 exec $RUSTC $RUSTFLAGS --jit $0 +PROFILE=$1 OUTPUT=$2 exec $RUSTC $RUSTFLAGS -Cllvm-args=mode=jit -Cprefer-dynamic $0 #*/ //! This program filters away uninteresting samples and trims uninteresting frames for stackcollapse diff --git a/compiler/rustc_codegen_cranelift/scripts/tests.sh b/compiler/rustc_codegen_cranelift/scripts/tests.sh index 114b6f30a4a91..a61774f479ec7 100755 --- a/compiler/rustc_codegen_cranelift/scripts/tests.sh +++ b/compiler/rustc_codegen_cranelift/scripts/tests.sh @@ -15,7 +15,10 @@ function no_sysroot_tests() { if [[ "$JIT_SUPPORTED" = "1" ]]; then echo "[JIT] mini_core_hello_world" - CG_CLIF_JIT_ARGS="abc bcd" $MY_RUSTC --jit example/mini_core_hello_world.rs --cfg jit --target "$HOST_TRIPLE" + CG_CLIF_JIT_ARGS="abc bcd" $MY_RUSTC -Cllvm-args=mode=jit -Cprefer-dynamic example/mini_core_hello_world.rs --cfg jit --target "$HOST_TRIPLE" + + echo "[JIT-lazy] mini_core_hello_world" + CG_CLIF_JIT_ARGS="abc bcd" $MY_RUSTC -Cllvm-args=mode=jit-lazy -Cprefer-dynamic example/mini_core_hello_world.rs --cfg jit --target "$HOST_TRIPLE" else echo "[JIT] mini_core_hello_world (skipped)" fi @@ -37,7 +40,10 @@ function base_sysroot_tests() { if [[ "$JIT_SUPPORTED" = "1" ]]; then echo "[JIT] std_example" - $MY_RUSTC --jit example/std_example.rs --target "$HOST_TRIPLE" + $MY_RUSTC -Cllvm-args=mode=jit -Cprefer-dynamic example/std_example.rs --target "$HOST_TRIPLE" + + echo "[JIT-lazy] std_example" + $MY_RUSTC -Cllvm-args=mode=jit-lazy -Cprefer-dynamic example/std_example.rs --cfg lazy_jit --target "$HOST_TRIPLE" else echo "[JIT] std_example (skipped)" fi diff --git a/compiler/rustc_codegen_cranelift/src/backend.rs b/compiler/rustc_codegen_cranelift/src/backend.rs index 9e32259716f51..0ce34c904bdcc 100644 --- a/compiler/rustc_codegen_cranelift/src/backend.rs +++ b/compiler/rustc_codegen_cranelift/src/backend.rs @@ -162,7 +162,7 @@ impl AddConstructor for ObjectProduct { } pub(crate) fn with_object(sess: &Session, name: &str, f: impl FnOnce(&mut Object)) -> Vec { - let triple = crate::build_isa(sess, true).triple().clone(); + let triple = crate::build_isa(sess).triple().clone(); let binary_format = match triple.binary_format { target_lexicon::BinaryFormat::Elf => object::BinaryFormat::Elf, @@ -193,7 +193,7 @@ pub(crate) fn with_object(sess: &Session, name: &str, f: impl FnOnce(&mut Object pub(crate) fn make_module(sess: &Session, name: String) -> ObjectModule { let mut builder = ObjectBuilder::new( - crate::build_isa(sess, true), + crate::build_isa(sess), name + ".o", cranelift_module::default_libcall_names(), ) diff --git a/compiler/rustc_codegen_cranelift/src/base.rs b/compiler/rustc_codegen_cranelift/src/base.rs index 72073896a723b..34c9561d67622 100644 --- a/compiler/rustc_codegen_cranelift/src/base.rs +++ b/compiler/rustc_codegen_cranelift/src/base.rs @@ -118,6 +118,8 @@ pub(crate) fn codegen_fn<'tcx>( context.eliminate_unreachable_code(cx.module.isa()).unwrap(); context.dce(cx.module.isa()).unwrap(); + context.want_disasm = crate::pretty_clif::should_write_ir(tcx); + // Define function let module = &mut cx.module; tcx.sess.time("define function", || { @@ -140,6 +142,16 @@ pub(crate) fn codegen_fn<'tcx>( &clif_comments, ); + if let Some(mach_compile_result) = &context.mach_compile_result { + if let Some(disasm) = &mach_compile_result.disasm { + crate::pretty_clif::write_ir_file( + tcx, + &format!("{}.vcode", tcx.symbol_name(instance).name), + |file| file.write_all(disasm.as_bytes()), + ) + } + } + // Define debuginfo for function let isa = cx.module.isa(); let debug_context = &mut cx.debug_context; @@ -307,7 +319,9 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, impl Module>) { } => { let discr = codegen_operand(fx, discr).load_scalar(fx); - if switch_ty.kind() == fx.tcx.types.bool.kind() { + let use_bool_opt = switch_ty.kind() == fx.tcx.types.bool.kind() + || (targets.iter().count() == 1 && targets.iter().next().unwrap().0 == 0); + if use_bool_opt { assert_eq!(targets.iter().count(), 1); let (then_value, then_block) = targets.iter().next().unwrap(); let then_block = fx.get_block(then_block); @@ -325,12 +339,22 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, impl Module>) { let discr = crate::optimize::peephole::maybe_unwrap_bint(&mut fx.bcx, discr); let discr = crate::optimize::peephole::make_branchable_value(&mut fx.bcx, discr); - if test_zero { - fx.bcx.ins().brz(discr, then_block, &[]); - fx.bcx.ins().jump(else_block, &[]); + if let Some(taken) = crate::optimize::peephole::maybe_known_branch_taken( + &fx.bcx, discr, test_zero, + ) { + if taken { + fx.bcx.ins().jump(then_block, &[]); + } else { + fx.bcx.ins().jump(else_block, &[]); + } } else { - fx.bcx.ins().brnz(discr, then_block, &[]); - fx.bcx.ins().jump(else_block, &[]); + if test_zero { + fx.bcx.ins().brz(discr, then_block, &[]); + fx.bcx.ins().jump(else_block, &[]); + } else { + fx.bcx.ins().brnz(discr, then_block, &[]); + fx.bcx.ins().jump(else_block, &[]); + } } } else { let mut switch = ::cranelift_frontend::Switch::new(); diff --git a/compiler/rustc_codegen_cranelift/src/bin/cg_clif.rs b/compiler/rustc_codegen_cranelift/src/bin/cg_clif.rs index f4d23ebcf4e4d..58e45b4e9b972 100644 --- a/compiler/rustc_codegen_cranelift/src/bin/cg_clif.rs +++ b/compiler/rustc_codegen_cranelift/src/bin/cg_clif.rs @@ -44,9 +44,7 @@ fn main() { let mut callbacks = CraneliftPassesCallbacks::default(); rustc_driver::install_ice_hook(); let exit_code = rustc_driver::catch_with_exit_code(|| { - let mut use_jit = false; - - let mut args = std::env::args_os() + let args = std::env::args_os() .enumerate() .map(|(i, arg)| { arg.into_string().unwrap_or_else(|arg| { @@ -56,23 +54,10 @@ fn main() { ) }) }) - .filter(|arg| { - if arg == "--jit" { - use_jit = true; - false - } else { - true - } - }) .collect::>(); - if use_jit { - args.push("-Cprefer-dynamic".to_string()); - } let mut run_compiler = rustc_driver::RunCompiler::new(&args, &mut callbacks); run_compiler.set_make_codegen_backend(Some(Box::new(move |_| { - Box::new(rustc_codegen_cranelift::CraneliftCodegenBackend { - config: rustc_codegen_cranelift::BackendConfig { use_jit }, - }) + Box::new(rustc_codegen_cranelift::CraneliftCodegenBackend { config: None }) }))); run_compiler.run() }); diff --git a/compiler/rustc_codegen_cranelift/src/bin/cg_clif_build_sysroot.rs b/compiler/rustc_codegen_cranelift/src/bin/cg_clif_build_sysroot.rs index 165d33dcfb509..8ee4cd46c94e0 100644 --- a/compiler/rustc_codegen_cranelift/src/bin/cg_clif_build_sysroot.rs +++ b/compiler/rustc_codegen_cranelift/src/bin/cg_clif_build_sysroot.rs @@ -92,9 +92,7 @@ fn main() { let mut run_compiler = rustc_driver::RunCompiler::new(&args, &mut callbacks); if use_clif { run_compiler.set_make_codegen_backend(Some(Box::new(move |_| { - Box::new(rustc_codegen_cranelift::CraneliftCodegenBackend { - config: rustc_codegen_cranelift::BackendConfig { use_jit: false }, - }) + Box::new(rustc_codegen_cranelift::CraneliftCodegenBackend { config: None }) }))); } run_compiler.run() diff --git a/compiler/rustc_codegen_cranelift/src/constant.rs b/compiler/rustc_codegen_cranelift/src/constant.rs index 544b020b71190..beff84fb2e217 100644 --- a/compiler/rustc_codegen_cranelift/src/constant.rs +++ b/compiler/rustc_codegen_cranelift/src/constant.rs @@ -100,7 +100,10 @@ fn codegen_static_ref<'tcx>( let global_ptr = fx.bcx.ins().global_value(fx.pointer_type, local_data_id); assert!(!layout.is_unsized(), "unsized statics aren't supported"); assert!( - matches!(fx.bcx.func.global_values[local_data_id], GlobalValueData::Symbol { tls: false, ..}), + matches!( + fx.bcx.func.global_values[local_data_id], + GlobalValueData::Symbol { tls: false, .. } + ), "tls static referenced without Rvalue::ThreadLocalRef" ); CPlace::for_ptr(crate::pointer::Pointer::new(global_ptr), layout) @@ -447,7 +450,8 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut impl Module, cx: &mut Constan data_ctx.write_data_addr(offset.bytes() as u32, global_value, addend as i64); } - module.define_data(data_id, &data_ctx).unwrap(); + // FIXME don't duplicate definitions in lazy jit mode + let _ = module.define_data(data_id, &data_ctx); cx.done.insert(data_id); } diff --git a/compiler/rustc_codegen_cranelift/src/debuginfo/emit.rs b/compiler/rustc_codegen_cranelift/src/debuginfo/emit.rs index c21835b1fc3aa..6160f9b78d8b3 100644 --- a/compiler/rustc_codegen_cranelift/src/debuginfo/emit.rs +++ b/compiler/rustc_codegen_cranelift/src/debuginfo/emit.rs @@ -74,10 +74,7 @@ impl WriterRelocate { /// Perform the collected relocations to be usable for JIT usage. #[cfg(feature = "jit")] - pub(super) fn relocate_for_jit( - mut self, - jit_module: &cranelift_simplejit::SimpleJITModule, - ) -> Vec { + pub(super) fn relocate_for_jit(mut self, jit_module: &cranelift_jit::JITModule) -> Vec { use std::convert::TryInto; for reloc in self.relocs.drain(..) { diff --git a/compiler/rustc_codegen_cranelift/src/debuginfo/unwind.rs b/compiler/rustc_codegen_cranelift/src/debuginfo/unwind.rs index e0f62b64e6bbb..49de927cdba05 100644 --- a/compiler/rustc_codegen_cranelift/src/debuginfo/unwind.rs +++ b/compiler/rustc_codegen_cranelift/src/debuginfo/unwind.rs @@ -15,11 +15,11 @@ pub(crate) struct UnwindContext<'tcx> { } impl<'tcx> UnwindContext<'tcx> { - pub(crate) fn new(tcx: TyCtxt<'tcx>, isa: &dyn TargetIsa) -> Self { + pub(crate) fn new(tcx: TyCtxt<'tcx>, isa: &dyn TargetIsa, pic_eh_frame: bool) -> Self { let mut frame_table = FrameTable::default(); let cie_id = if let Some(mut cie) = isa.create_systemv_cie() { - if isa.flags().is_pic() { + if pic_eh_frame { cie.fde_address_encoding = gimli::DwEhPe(gimli::DW_EH_PE_pcrel.0 | gimli::DW_EH_PE_sdata4.0); } @@ -80,7 +80,7 @@ impl<'tcx> UnwindContext<'tcx> { #[cfg(feature = "jit")] pub(crate) unsafe fn register_jit( self, - jit_module: &cranelift_simplejit::SimpleJITModule, + jit_module: &cranelift_jit::JITModule, ) -> Option { let mut eh_frame = EhFrame::from(super::emit::WriterRelocate::new(super::target_endian( self.tcx, diff --git a/compiler/rustc_codegen_cranelift/src/driver/aot.rs b/compiler/rustc_codegen_cranelift/src/driver/aot.rs index 78d6ff0cb001c..16f9bfc99189f 100644 --- a/compiler/rustc_codegen_cranelift/src/driver/aot.rs +++ b/compiler/rustc_codegen_cranelift/src/driver/aot.rs @@ -8,7 +8,7 @@ use rustc_codegen_ssa::{CodegenResults, CompiledModule, CrateInfo, ModuleKind}; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_middle::dep_graph::{WorkProduct, WorkProductId}; use rustc_middle::middle::cstore::EncodedMetadata; -use rustc_middle::mir::mono::CodegenUnit; +use rustc_middle::mir::mono::{CodegenUnit, MonoItem}; use rustc_session::cgu_reuse_tracker::CguReuse; use rustc_session::config::{DebugInfo, OutputType}; @@ -146,11 +146,34 @@ fn module_codegen(tcx: TyCtxt<'_>, cgu_name: rustc_span::Symbol) -> ModuleCodege } } - let mut cx = crate::CodegenCx::new(tcx, module, tcx.sess.opts.debuginfo != DebugInfo::None); + let mut cx = crate::CodegenCx::new( + tcx, + module, + tcx.sess.opts.debuginfo != DebugInfo::None, + true, + ); super::predefine_mono_items(&mut cx, &mono_items); for (mono_item, (linkage, visibility)) in mono_items { let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility); - super::codegen_mono_item(&mut cx, mono_item, linkage); + match mono_item { + MonoItem::Fn(inst) => { + cx.tcx.sess.time("codegen fn", || { + crate::base::codegen_fn(&mut cx, inst, linkage) + }); + } + MonoItem::Static(def_id) => { + crate::constant::codegen_static(&mut cx.constants_cx, def_id) + } + MonoItem::GlobalAsm(hir_id) => { + let item = cx.tcx.hir().expect_item(hir_id); + if let rustc_hir::ItemKind::GlobalAsm(rustc_hir::GlobalAsm { asm }) = item.kind { + cx.global_asm.push_str(&*asm.as_str()); + cx.global_asm.push_str("\n\n"); + } else { + bug!("Expected GlobalAsm found {:?}", item); + } + } + } } let (mut module, global_asm, debug, mut unwind_context) = tcx.sess.time("finalize CodegenCx", || cx.finalize()); @@ -236,7 +259,7 @@ pub(super) fn run_aot( tcx.sess.abort_if_errors(); let mut allocator_module = new_module(tcx, "allocator_shim".to_string()); - let mut allocator_unwind_context = UnwindContext::new(tcx, allocator_module.isa()); + let mut allocator_unwind_context = UnwindContext::new(tcx, allocator_module.isa(), true); let created_alloc_shim = crate::allocator::codegen(tcx, &mut allocator_module, &mut allocator_unwind_context); diff --git a/compiler/rustc_codegen_cranelift/src/driver/jit.rs b/compiler/rustc_codegen_cranelift/src/driver/jit.rs index 5a844841c2ce5..9a42c675cc144 100644 --- a/compiler/rustc_codegen_cranelift/src/driver/jit.rs +++ b/compiler/rustc_codegen_cranelift/src/driver/jit.rs @@ -1,16 +1,23 @@ //! The JIT driver uses [`cranelift_simplejit`] to JIT execute programs without writing any object //! files. +use std::cell::RefCell; use std::ffi::CString; use std::os::raw::{c_char, c_int}; use rustc_codegen_ssa::CrateInfo; +use rustc_middle::mir::mono::MonoItem; -use cranelift_simplejit::{SimpleJITBuilder, SimpleJITModule}; +use cranelift_jit::{JITBuilder, JITModule}; use crate::prelude::*; +use crate::{CodegenCx, CodegenMode}; -pub(super) fn run_jit(tcx: TyCtxt<'_>) -> ! { +thread_local! { + pub static CURRENT_MODULE: RefCell> = RefCell::new(None); +} + +pub(super) fn run_jit(tcx: TyCtxt<'_>, codegen_mode: CodegenMode) -> ! { if !tcx.sess.opts.output_types.should_codegen() { tcx.sess.fatal("JIT mode doesn't work with `cargo check`."); } @@ -35,12 +42,13 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>) -> ! { let imported_symbols = load_imported_symbols_for_jit(tcx); - let mut jit_builder = SimpleJITBuilder::with_isa( - crate::build_isa(tcx.sess, false), + let mut jit_builder = JITBuilder::with_isa( + crate::build_isa(tcx.sess), cranelift_module::default_libcall_names(), ); + jit_builder.hotswap(matches!(codegen_mode, CodegenMode::JitLazy)); jit_builder.symbols(imported_symbols); - let mut jit_module = SimpleJITModule::new(jit_builder); + let mut jit_module = JITModule::new(jit_builder); assert_eq!(pointer_ty(tcx), jit_module.target_config().pointer_type()); let sig = Signature { @@ -66,20 +74,42 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>) -> ! { .into_iter() .collect::>(); - let mut cx = crate::CodegenCx::new(tcx, jit_module, false); + let mut cx = crate::CodegenCx::new(tcx, jit_module, false, false); - let (mut jit_module, global_asm, _debug, mut unwind_context) = - super::time(tcx, "codegen mono items", || { - super::predefine_mono_items(&mut cx, &mono_items); - for (mono_item, (linkage, visibility)) in mono_items { - let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility); - super::codegen_mono_item(&mut cx, mono_item, linkage); + super::time(tcx, "codegen mono items", || { + super::predefine_mono_items(&mut cx, &mono_items); + for (mono_item, (linkage, visibility)) in mono_items { + let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility); + match mono_item { + MonoItem::Fn(inst) => match codegen_mode { + CodegenMode::Aot => unreachable!(), + CodegenMode::Jit => { + cx.tcx.sess.time("codegen fn", || { + crate::base::codegen_fn(&mut cx, inst, linkage) + }); + } + CodegenMode::JitLazy => codegen_shim(&mut cx, inst), + }, + MonoItem::Static(def_id) => { + crate::constant::codegen_static(&mut cx.constants_cx, def_id); + } + MonoItem::GlobalAsm(hir_id) => { + let item = cx.tcx.hir().expect_item(hir_id); + tcx.sess + .span_fatal(item.span, "Global asm is not supported in JIT mode"); + } } - tcx.sess.time("finalize CodegenCx", || cx.finalize()) - }); + } + }); + + let (mut jit_module, global_asm, _debug, mut unwind_context) = + tcx.sess.time("finalize CodegenCx", || cx.finalize()); + jit_module.finalize_definitions(); + if !global_asm.is_empty() { - tcx.sess.fatal("Global asm is not supported in JIT mode"); + tcx.sess.fatal("Inline asm is not supported in JIT mode"); } + crate::main_shim::maybe_create_entry_wrapper(tcx, &mut jit_module, &mut unwind_context, true); crate::allocator::codegen(tcx, &mut jit_module, &mut unwind_context); @@ -91,7 +121,7 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>) -> ! { let finalized_main: *const u8 = jit_module.get_finalized_function(main_func_id); - println!("Rustc codegen cranelift will JIT run the executable, because --jit was passed"); + println!("Rustc codegen cranelift will JIT run the executable, because -Cllvm-args=mode=jit was passed"); let f: extern "C" fn(c_int, *const *const c_char) -> c_int = unsafe { ::std::mem::transmute(finalized_main) }; @@ -107,11 +137,50 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>) -> ! { // useful as some dynamic linkers use it as a marker to jump over. argv.push(std::ptr::null()); + CURRENT_MODULE + .with(|current_module| assert!(current_module.borrow_mut().replace(jit_module).is_none())); + let ret = f(args.len() as c_int, argv.as_ptr()); std::process::exit(ret); } +#[no_mangle] +extern "C" fn __clif_jit_fn(instance_ptr: *const Instance<'static>) -> *const u8 { + rustc_middle::ty::tls::with(|tcx| { + // lift is used to ensure the correct lifetime for instance. + let instance = tcx.lift(unsafe { *instance_ptr }).unwrap(); + + CURRENT_MODULE.with(|jit_module| { + let mut jit_module = jit_module.borrow_mut(); + let jit_module = jit_module.as_mut().unwrap(); + let mut cx = crate::CodegenCx::new(tcx, jit_module, false, false); + + let (name, sig) = crate::abi::get_function_name_and_sig( + tcx, + cx.module.isa().triple(), + instance, + true, + ); + let func_id = cx + .module + .declare_function(&name, Linkage::Export, &sig) + .unwrap(); + cx.module.prepare_for_function_redefine(func_id).unwrap(); + + tcx.sess.time("codegen fn", || { + crate::base::codegen_fn(&mut cx, instance, Linkage::Export) + }); + + let (jit_module, global_asm, _debug_context, unwind_context) = cx.finalize(); + assert!(global_asm.is_empty()); + jit_module.finalize_definitions(); + std::mem::forget(unsafe { unwind_context.register_jit(&jit_module) }); + jit_module.get_finalized_function(func_id) + }) + }) +} + fn load_imported_symbols_for_jit(tcx: TyCtxt<'_>) -> Vec<(String, *const u8)> { use rustc_middle::middle::dependency_format::Linkage; @@ -171,3 +240,68 @@ fn load_imported_symbols_for_jit(tcx: TyCtxt<'_>) -> Vec<(String, *const u8)> { imported_symbols } + +pub(super) fn codegen_shim<'tcx>(cx: &mut CodegenCx<'tcx, impl Module>, inst: Instance<'tcx>) { + let tcx = cx.tcx; + + let pointer_type = cx.module.target_config().pointer_type(); + + let (name, sig) = + crate::abi::get_function_name_and_sig(tcx, cx.module.isa().triple(), inst, true); + let func_id = cx + .module + .declare_function(&name, Linkage::Export, &sig) + .unwrap(); + + let instance_ptr = Box::into_raw(Box::new(inst)); + + let jit_fn = cx + .module + .declare_function( + "__clif_jit_fn", + Linkage::Import, + &Signature { + call_conv: cx.module.target_config().default_call_conv, + params: vec![AbiParam::new(pointer_type)], + returns: vec![AbiParam::new(pointer_type)], + }, + ) + .unwrap(); + + let mut trampoline = Function::with_name_signature(ExternalName::default(), sig.clone()); + let mut builder_ctx = FunctionBuilderContext::new(); + let mut trampoline_builder = FunctionBuilder::new(&mut trampoline, &mut builder_ctx); + + let jit_fn = cx + .module + .declare_func_in_func(jit_fn, trampoline_builder.func); + let sig_ref = trampoline_builder.func.import_signature(sig); + + let entry_block = trampoline_builder.create_block(); + trampoline_builder.append_block_params_for_function_params(entry_block); + let fn_args = trampoline_builder + .func + .dfg + .block_params(entry_block) + .to_vec(); + + trampoline_builder.switch_to_block(entry_block); + let instance_ptr = trampoline_builder + .ins() + .iconst(pointer_type, instance_ptr as u64 as i64); + let jitted_fn = trampoline_builder.ins().call(jit_fn, &[instance_ptr]); + let jitted_fn = trampoline_builder.func.dfg.inst_results(jitted_fn)[0]; + let call_inst = trampoline_builder + .ins() + .call_indirect(sig_ref, jitted_fn, &fn_args); + let ret_vals = trampoline_builder.func.dfg.inst_results(call_inst).to_vec(); + trampoline_builder.ins().return_(&ret_vals); + + cx.module + .define_function( + func_id, + &mut Context::for_function(trampoline), + &mut cranelift_codegen::binemit::NullTrapSink {}, + ) + .unwrap(); +} diff --git a/compiler/rustc_codegen_cranelift/src/driver/mod.rs b/compiler/rustc_codegen_cranelift/src/driver/mod.rs index 7b8cc2ddd48d6..9f4ea9a386551 100644 --- a/compiler/rustc_codegen_cranelift/src/driver/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/driver/mod.rs @@ -7,6 +7,7 @@ use rustc_middle::middle::cstore::EncodedMetadata; use rustc_middle::mir::mono::{Linkage as RLinkage, MonoItem, Visibility}; use crate::prelude::*; +use crate::CodegenMode; mod aot; #[cfg(feature = "jit")] @@ -20,24 +21,25 @@ pub(crate) fn codegen_crate( ) -> Box { tcx.sess.abort_if_errors(); - if config.use_jit { - let is_executable = tcx - .sess - .crate_types() - .contains(&rustc_session::config::CrateType::Executable); - if !is_executable { - tcx.sess.fatal("can't jit non-executable crate"); - } + match config.codegen_mode { + CodegenMode::Aot => aot::run_aot(tcx, metadata, need_metadata_module), + CodegenMode::Jit | CodegenMode::JitLazy => { + let is_executable = tcx + .sess + .crate_types() + .contains(&rustc_session::config::CrateType::Executable); + if !is_executable { + tcx.sess.fatal("can't jit non-executable crate"); + } - #[cfg(feature = "jit")] - let _: ! = jit::run_jit(tcx); + #[cfg(feature = "jit")] + let _: ! = jit::run_jit(tcx, config.codegen_mode); - #[cfg(not(feature = "jit"))] - tcx.sess - .fatal("jit support was disabled when compiling rustc_codegen_cranelift"); + #[cfg(not(feature = "jit"))] + tcx.sess + .fatal("jit support was disabled when compiling rustc_codegen_cranelift"); + } } - - aot::run_aot(tcx, metadata, need_metadata_module) } fn predefine_mono_items<'tcx>( @@ -63,30 +65,6 @@ fn predefine_mono_items<'tcx>( }); } -fn codegen_mono_item<'tcx, M: Module>( - cx: &mut crate::CodegenCx<'tcx, M>, - mono_item: MonoItem<'tcx>, - linkage: Linkage, -) { - match mono_item { - MonoItem::Fn(inst) => { - cx.tcx - .sess - .time("codegen fn", || crate::base::codegen_fn(cx, inst, linkage)); - } - MonoItem::Static(def_id) => crate::constant::codegen_static(&mut cx.constants_cx, def_id), - MonoItem::GlobalAsm(hir_id) => { - let item = cx.tcx.hir().expect_item(hir_id); - if let rustc_hir::ItemKind::GlobalAsm(rustc_hir::GlobalAsm { asm }) = item.kind { - cx.global_asm.push_str(&*asm.as_str()); - cx.global_asm.push_str("\n\n"); - } else { - bug!("Expected GlobalAsm found {:?}", item); - } - } - } -} - fn time(tcx: TyCtxt<'_>, name: &'static str, f: impl FnOnce() -> R) -> R { if std::env::var("CG_CLIF_DISPLAY_CG_TIME") .as_ref() diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs index 171445f2d71b6..d58e4d4995842 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs @@ -23,8 +23,8 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>( // Used by `_mm_movemask_epi8` and `_mm256_movemask_epi8` llvm.x86.sse2.pmovmskb.128 | llvm.x86.avx2.pmovmskb | llvm.x86.sse2.movmsk.pd, (c a) { - let (lane_layout, lane_count) = lane_type_and_count(fx.tcx, a.layout()); - let lane_ty = fx.clif_type(lane_layout.ty).unwrap(); + let (lane_count, lane_ty) = a.layout().ty.simd_size_and_type(fx.tcx); + let lane_ty = fx.clif_type(lane_ty).unwrap(); assert!(lane_count <= 32); let mut res = fx.bcx.ins().iconst(types::I32, 0); diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs index df8aa1b3e6983..be5b247bb9f0b 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs @@ -171,27 +171,6 @@ macro validate_simd_type($fx:ident, $intrinsic:ident, $span:ident, $ty:expr) { } } -fn lane_type_and_count<'tcx>( - tcx: TyCtxt<'tcx>, - layout: TyAndLayout<'tcx>, -) -> (TyAndLayout<'tcx>, u16) { - assert!(layout.ty.is_simd()); - let lane_count = match layout.fields { - rustc_target::abi::FieldsShape::Array { stride: _, count } => u16::try_from(count).unwrap(), - _ => unreachable!("lane_type_and_count({:?})", layout), - }; - let lane_layout = layout - .field( - &ty::layout::LayoutCx { - tcx, - param_env: ParamEnv::reveal_all(), - }, - 0, - ) - .unwrap(); - (lane_layout, lane_count) -} - pub(crate) fn clif_vector_type<'tcx>(tcx: TyCtxt<'tcx>, layout: TyAndLayout<'tcx>) -> Option { let (element, count) = match &layout.abi { Abi::Vector { element, count } => (element.clone(), *count), @@ -218,8 +197,10 @@ fn simd_for_each_lane<'tcx, M: Module>( ) { let layout = val.layout(); - let (lane_layout, lane_count) = lane_type_and_count(fx.tcx, layout); - let (ret_lane_layout, ret_lane_count) = lane_type_and_count(fx.tcx, ret.layout()); + let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx); + let lane_layout = fx.layout_of(lane_ty); + let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx); + let ret_lane_layout = fx.layout_of(ret_lane_ty); assert_eq!(lane_count, ret_lane_count); for lane_idx in 0..lane_count { @@ -248,8 +229,10 @@ fn simd_pair_for_each_lane<'tcx, M: Module>( assert_eq!(x.layout(), y.layout()); let layout = x.layout(); - let (lane_layout, lane_count) = lane_type_and_count(fx.tcx, layout); - let (ret_lane_layout, ret_lane_count) = lane_type_and_count(fx.tcx, ret.layout()); + let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx); + let lane_layout = fx.layout_of(lane_ty); + let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx); + let ret_lane_layout = fx.layout_of(ret_lane_ty); assert_eq!(lane_count, ret_lane_count); for lane in 0..lane_count { @@ -269,13 +252,14 @@ fn simd_reduce<'tcx, M: Module>( ret: CPlace<'tcx>, f: impl Fn(&mut FunctionCx<'_, 'tcx, M>, TyAndLayout<'tcx>, Value, Value) -> Value, ) { - let (lane_layout, lane_count) = lane_type_and_count(fx.tcx, val.layout()); + let (lane_count, lane_ty) = val.layout().ty.simd_size_and_type(fx.tcx); + let lane_layout = fx.layout_of(lane_ty); assert_eq!(lane_layout, ret.layout()); let mut res_val = val.value_field(fx, mir::Field::new(0)).load_scalar(fx); for lane_idx in 1..lane_count { let lane = val - .value_field(fx, mir::Field::new(lane_idx.into())) + .value_field(fx, mir::Field::new(lane_idx.try_into().unwrap())) .load_scalar(fx); res_val = f(fx, lane_layout, res_val, lane); } @@ -289,14 +273,14 @@ fn simd_reduce_bool<'tcx, M: Module>( ret: CPlace<'tcx>, f: impl Fn(&mut FunctionCx<'_, 'tcx, M>, Value, Value) -> Value, ) { - let (_lane_layout, lane_count) = lane_type_and_count(fx.tcx, val.layout()); + let (lane_count, _lane_ty) = val.layout().ty.simd_size_and_type(fx.tcx); assert!(ret.layout().ty.is_bool()); let res_val = val.value_field(fx, mir::Field::new(0)).load_scalar(fx); let mut res_val = fx.bcx.ins().band_imm(res_val, 1); // mask to boolean for lane_idx in 1..lane_count { let lane = val - .value_field(fx, mir::Field::new(lane_idx.into())) + .value_field(fx, mir::Field::new(lane_idx.try_into().unwrap())) .load_scalar(fx); let lane = fx.bcx.ins().band_imm(lane, 1); // mask to boolean res_val = f(fx, res_val, lane); @@ -460,9 +444,6 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( "abort" => { trap_abort(fx, "Called intrinsic::abort."); } - "unreachable" => { - trap_unreachable(fx, "[corruption] Called intrinsic::unreachable."); - } "transmute" => { crate::base::codegen_panic(fx, "Transmuting to uninhabited type.", span); } @@ -575,12 +556,6 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( fx.bcx.call_memmove(fx.cx.module.target_config(), dst, src, byte_amount); } }; - discriminant_value, (c ptr) { - let pointee_layout = fx.layout_of(ptr.layout().ty.builtin_deref(true).unwrap().ty); - let val = CValue::by_ref(Pointer::new(ptr.load_scalar(fx)), pointee_layout); - let discr = crate::discriminant::codegen_get_discriminant(fx, val, ret.layout()); - ret.write_cvalue(fx, discr); - }; size_of_val, (c ptr) { let layout = fx.layout_of(T); let size = if layout.is_unsized() { @@ -641,22 +616,6 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( ); ret.write_cvalue(fx, res); }; - _ if intrinsic.starts_with("wrapping_"), (c x, c y) { - assert_eq!(x.layout().ty, y.layout().ty); - let bin_op = match intrinsic { - "wrapping_add" => BinOp::Add, - "wrapping_sub" => BinOp::Sub, - "wrapping_mul" => BinOp::Mul, - _ => unreachable!("intrinsic {}", intrinsic), - }; - let res = crate::num::codegen_int_binop( - fx, - bin_op, - x, - y, - ); - ret.write_cvalue(fx, res); - }; _ if intrinsic.starts_with("saturating_"), (c lhs, c rhs) { assert_eq!(lhs.layout().ty, rhs.layout().ty); let bin_op = match intrinsic { @@ -916,7 +875,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( dest.write_cvalue(fx, val); }; - size_of | pref_align_of | min_align_of | needs_drop | type_id | type_name | variant_count, () { + pref_align_of | min_align_of | needs_drop | type_id | type_name | variant_count, () { let const_val = fx.tcx.const_eval_instance(ParamEnv::reveal_all(), instance, None).unwrap(); let val = crate::constant::codegen_const_value( diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs index 2b32e866e5ef6..e0eb5c59590ff 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs @@ -73,11 +73,11 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( assert_eq!(x.layout(), y.layout()); let layout = x.layout(); - let (lane_type, lane_count) = lane_type_and_count(fx.tcx, layout); - let (ret_lane_type, ret_lane_count) = lane_type_and_count(fx.tcx, ret.layout()); + let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx); + let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx); - assert_eq!(lane_type, ret_lane_type); - assert_eq!(n, ret_lane_count); + assert_eq!(lane_ty, ret_lane_ty); + assert_eq!(u64::from(n), ret_lane_count); let total_len = lane_count * 2; @@ -105,14 +105,14 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( }; for &idx in &indexes { - assert!(idx < total_len, "idx {} out of range 0..{}", idx, total_len); + assert!(u64::from(idx) < total_len, "idx {} out of range 0..{}", idx, total_len); } for (out_idx, in_idx) in indexes.into_iter().enumerate() { - let in_lane = if in_idx < lane_count { + let in_lane = if u64::from(in_idx) < lane_count { x.value_field(fx, mir::Field::new(in_idx.into())) } else { - y.value_field(fx, mir::Field::new((in_idx - lane_count).into())) + y.value_field(fx, mir::Field::new(usize::from(in_idx) - usize::try_from(lane_count).unwrap())) }; let out_lane = ret.place_field(fx, mir::Field::new(out_idx)); out_lane.write_cvalue(fx, in_lane); @@ -131,7 +131,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( }; let idx = idx_const.val.try_to_bits(Size::from_bytes(4 /* u32*/)).unwrap_or_else(|| panic!("kind not scalar: {:?}", idx_const)); - let (_lane_type, lane_count) = lane_type_and_count(fx.tcx, base.layout()); + let (lane_count, _lane_ty) = base.layout().ty.simd_size_and_type(fx.tcx); if idx >= lane_count.into() { fx.tcx.sess.span_fatal(fx.mir.span, &format!("[simd_insert] idx {} >= lane_count {}", idx, lane_count)); } @@ -160,7 +160,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( }; let idx = idx_const.val.try_to_bits(Size::from_bytes(4 /* u32*/)).unwrap_or_else(|| panic!("kind not scalar: {:?}", idx_const)); - let (_lane_type, lane_count) = lane_type_and_count(fx.tcx, v.layout()); + let (lane_count, _lane_ty) = v.layout().ty.simd_size_and_type(fx.tcx); if idx >= lane_count.into() { fx.tcx.sess.span_fatal(fx.mir.span, &format!("[simd_extract] idx {} >= lane_count {}", idx, lane_count)); } @@ -212,12 +212,13 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( assert_eq!(a.layout(), c.layout()); let layout = a.layout(); - let (_lane_layout, lane_count) = lane_type_and_count(fx.tcx, layout); - let (ret_lane_layout, ret_lane_count) = lane_type_and_count(fx.tcx, ret.layout()); + let (lane_count, _lane_ty) = layout.ty.simd_size_and_type(fx.tcx); + let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx); assert_eq!(lane_count, ret_lane_count); + let ret_lane_layout = fx.layout_of(ret_lane_ty); for lane in 0..lane_count { - let lane = mir::Field::new(lane.into()); + let lane = mir::Field::new(lane.try_into().unwrap()); let a_lane = a.value_field(fx, lane).load_scalar(fx); let b_lane = b.value_field(fx, lane).load_scalar(fx); let c_lane = c.value_field(fx, lane).load_scalar(fx); diff --git a/compiler/rustc_codegen_cranelift/src/lib.rs b/compiler/rustc_codegen_cranelift/src/lib.rs index ba9ee0d450ee6..6e4f3bf2898d8 100644 --- a/compiler/rustc_codegen_cranelift/src/lib.rs +++ b/compiler/rustc_codegen_cranelift/src/lib.rs @@ -5,7 +5,8 @@ associated_type_bounds, never_type, try_blocks, - hash_drain_filter + hash_drain_filter, + str_split_once )] #![warn(rust_2018_idioms)] #![warn(unused_lifetimes)] @@ -34,6 +35,7 @@ extern crate rustc_target; extern crate rustc_driver; use std::any::Any; +use std::str::FromStr; use rustc_codegen_ssa::traits::CodegenBackend; use rustc_codegen_ssa::CodegenResults; @@ -141,8 +143,8 @@ struct CodegenCx<'tcx, M: Module> { } impl<'tcx, M: Module> CodegenCx<'tcx, M> { - fn new(tcx: TyCtxt<'tcx>, module: M, debug_info: bool) -> Self { - let unwind_context = UnwindContext::new(tcx, module.isa()); + fn new(tcx: TyCtxt<'tcx>, module: M, debug_info: bool, pic_eh_frame: bool) -> Self { + let unwind_context = UnwindContext::new(tcx, module.isa(), pic_eh_frame); let debug_context = if debug_info { Some(DebugContext::new(tcx, module.isa())) } else { @@ -172,12 +174,55 @@ impl<'tcx, M: Module> CodegenCx<'tcx, M> { } #[derive(Copy, Clone, Debug)] +pub enum CodegenMode { + Aot, + Jit, + JitLazy, +} + +impl Default for CodegenMode { + fn default() -> Self { + CodegenMode::Aot + } +} + +impl FromStr for CodegenMode { + type Err = String; + + fn from_str(s: &str) -> Result { + match s { + "aot" => Ok(CodegenMode::Aot), + "jit" => Ok(CodegenMode::Jit), + "jit-lazy" => Ok(CodegenMode::JitLazy), + _ => Err(format!("Unknown codegen mode `{}`", s)), + } + } +} + +#[derive(Copy, Clone, Debug, Default)] pub struct BackendConfig { - pub use_jit: bool, + pub codegen_mode: CodegenMode, +} + +impl BackendConfig { + fn from_opts(opts: &[String]) -> Result { + let mut config = BackendConfig::default(); + for opt in opts { + if let Some((name, value)) = opt.split_once('=') { + match name { + "mode" => config.codegen_mode = value.parse()?, + _ => return Err(format!("Unknown option `{}`", name)), + } + } else { + return Err(format!("Invalid option `{}`", opt)); + } + } + Ok(config) + } } pub struct CraneliftCodegenBackend { - pub config: BackendConfig, + pub config: Option, } impl CodegenBackend for CraneliftCodegenBackend { @@ -204,7 +249,13 @@ impl CodegenBackend for CraneliftCodegenBackend { metadata: EncodedMetadata, need_metadata_module: bool, ) -> Box { - let res = driver::codegen_crate(tcx, metadata, need_metadata_module, self.config); + let config = if let Some(config) = self.config { + config + } else { + BackendConfig::from_opts(&tcx.sess.opts.cg.llvm_args) + .unwrap_or_else(|err| tcx.sess.fatal(&err)) + }; + let res = driver::codegen_crate(tcx, metadata, need_metadata_module, config); rustc_symbol_mangling::test::report_symbol_names(tcx); @@ -250,17 +301,13 @@ fn target_triple(sess: &Session) -> target_lexicon::Triple { sess.target.llvm_target.parse().unwrap() } -fn build_isa(sess: &Session, enable_pic: bool) -> Box { +fn build_isa(sess: &Session) -> Box { use target_lexicon::BinaryFormat; let target_triple = crate::target_triple(sess); let mut flags_builder = settings::builder(); - if enable_pic { - flags_builder.enable("is_pic").unwrap(); - } else { - flags_builder.set("is_pic", "false").unwrap(); - } + flags_builder.enable("is_pic").unwrap(); flags_builder.set("enable_probestack", "false").unwrap(); // __cranelift_probestack is not provided flags_builder .set( @@ -283,8 +330,6 @@ fn build_isa(sess: &Session, enable_pic: bool) -> Box { @@ -297,7 +342,7 @@ fn build_isa(sess: &Session, enable_pic: bool) -> Box { sess.warn("Optimizing for size is not supported. Just ignoring the request"); } - }*/ + } let flags = settings::Flags::new(flags_builder); @@ -311,7 +356,5 @@ fn build_isa(sess: &Session, enable_pic: bool) -> Box Box { - Box::new(CraneliftCodegenBackend { - config: BackendConfig { use_jit: false }, - }) + Box::new(CraneliftCodegenBackend { config: None }) } diff --git a/compiler/rustc_codegen_cranelift/src/optimize/peephole.rs b/compiler/rustc_codegen_cranelift/src/optimize/peephole.rs index f8e0f3af3d0ad..a575ed8dc35f8 100644 --- a/compiler/rustc_codegen_cranelift/src/optimize/peephole.rs +++ b/compiler/rustc_codegen_cranelift/src/optimize/peephole.rs @@ -73,7 +73,7 @@ pub(crate) fn make_branchable_value(bcx: &mut FunctionBuilder<'_>, arg: Value) - })() .unwrap_or_else(|| { match bcx.func.dfg.value_type(arg) { - types::I8 | types::I32 => { + types::I8 | types::I16 => { // WORKAROUND for brz.i8 and brnz.i8 not yet being implemented bcx.ins().uextend(types::I32, arg) } @@ -81,3 +81,40 @@ pub(crate) fn make_branchable_value(bcx: &mut FunctionBuilder<'_>, arg: Value) - } }) } + +/// Returns whether the branch is statically known to be taken or `None` if it isn't statically known. +pub(crate) fn maybe_known_branch_taken( + bcx: &FunctionBuilder<'_>, + arg: Value, + test_zero: bool, +) -> Option { + let arg_inst = if let ValueDef::Result(arg_inst, 0) = bcx.func.dfg.value_def(arg) { + arg_inst + } else { + return None; + }; + + match bcx.func.dfg[arg_inst] { + InstructionData::UnaryBool { + opcode: Opcode::Bconst, + imm, + } => { + if test_zero { + Some(!imm) + } else { + Some(imm) + } + } + InstructionData::UnaryImm { + opcode: Opcode::Iconst, + imm, + } => { + if test_zero { + Some(imm.bits() == 0) + } else { + Some(imm.bits() != 0) + } + } + _ => None, + } +} diff --git a/compiler/rustc_codegen_cranelift/src/pretty_clif.rs b/compiler/rustc_codegen_cranelift/src/pretty_clif.rs index a9f060e51d8f8..22c94fec82fc1 100644 --- a/compiler/rustc_codegen_cranelift/src/pretty_clif.rs +++ b/compiler/rustc_codegen_cranelift/src/pretty_clif.rs @@ -53,6 +53,7 @@ //! ``` use std::fmt; +use std::io::Write; use cranelift_codegen::{ entity::SecondaryMap, @@ -200,32 +201,24 @@ impl FunctionCx<'_, '_, M> { } } -pub(crate) fn write_clif_file<'tcx>( - tcx: TyCtxt<'tcx>, - postfix: &str, - isa: Option<&dyn cranelift_codegen::isa::TargetIsa>, - instance: Instance<'tcx>, - context: &cranelift_codegen::Context, - mut clif_comments: &CommentWriter, -) { - use std::io::Write; - - if !cfg!(debug_assertions) - && !tcx +pub(crate) fn should_write_ir(tcx: TyCtxt<'_>) -> bool { + cfg!(debug_assertions) + || tcx .sess .opts .output_types .contains_key(&OutputType::LlvmAssembly) - { +} + +pub(crate) fn write_ir_file<'tcx>( + tcx: TyCtxt<'tcx>, + name: &str, + write: impl FnOnce(&mut dyn Write) -> std::io::Result<()>, +) { + if !should_write_ir(tcx) { return; } - let value_ranges = isa.map(|isa| { - context - .build_value_labels_ranges(isa) - .expect("value location ranges") - }); - let clif_output_dir = tcx.output_filenames(LOCAL_CRATE).with_extension("clif"); match std::fs::create_dir(&clif_output_dir) { @@ -234,41 +227,58 @@ pub(crate) fn write_clif_file<'tcx>( res @ Err(_) => res.unwrap(), } - let clif_file_name = clif_output_dir.join(format!( - "{}.{}.clif", - tcx.symbol_name(instance).name, - postfix - )); - - let mut clif = String::new(); - cranelift_codegen::write::decorate_function( - &mut clif_comments, - &mut clif, - &context.func, - &DisplayFunctionAnnotations { - isa: Some(&*crate::build_isa( - tcx.sess, true, /* PIC doesn't matter here */ - )), - value_ranges: value_ranges.as_ref(), - }, - ) - .unwrap(); + let clif_file_name = clif_output_dir.join(name); let res: std::io::Result<()> = try { let mut file = std::fs::File::create(clif_file_name)?; - let target_triple = crate::target_triple(tcx.sess); - writeln!(file, "test compile")?; - writeln!(file, "set is_pic")?; - writeln!(file, "set enable_simd")?; - writeln!(file, "target {} haswell", target_triple)?; - writeln!(file)?; - file.write_all(clif.as_bytes())?; + write(&mut file)?; }; if let Err(err) = res { - tcx.sess.warn(&format!("err writing clif file: {}", err)); + tcx.sess.warn(&format!("error writing ir file: {}", err)); } } +pub(crate) fn write_clif_file<'tcx>( + tcx: TyCtxt<'tcx>, + postfix: &str, + isa: Option<&dyn cranelift_codegen::isa::TargetIsa>, + instance: Instance<'tcx>, + context: &cranelift_codegen::Context, + mut clif_comments: &CommentWriter, +) { + write_ir_file( + tcx, + &format!("{}.{}.clif", tcx.symbol_name(instance).name, postfix), + |file| { + let value_ranges = isa.map(|isa| { + context + .build_value_labels_ranges(isa) + .expect("value location ranges") + }); + + let mut clif = String::new(); + cranelift_codegen::write::decorate_function( + &mut clif_comments, + &mut clif, + &context.func, + &DisplayFunctionAnnotations { + isa: Some(&*crate::build_isa(tcx.sess)), + value_ranges: value_ranges.as_ref(), + }, + ) + .unwrap(); + + writeln!(file, "test compile")?; + writeln!(file, "set is_pic")?; + writeln!(file, "set enable_simd")?; + writeln!(file, "target {} haswell", crate::target_triple(tcx.sess))?; + writeln!(file)?; + file.write_all(clif.as_bytes())?; + Ok(()) + }, + ); +} + impl fmt::Debug for FunctionCx<'_, '_, M> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { writeln!(f, "{:?}", self.instance.substs)?; diff --git a/compiler/rustc_codegen_cranelift/src/vtable.rs b/compiler/rustc_codegen_cranelift/src/vtable.rs index 238abc0d8bdfa..8f15586a9dc06 100644 --- a/compiler/rustc_codegen_cranelift/src/vtable.rs +++ b/compiler/rustc_codegen_cranelift/src/vtable.rs @@ -158,7 +158,8 @@ fn build_vtable<'tcx>( ) .unwrap(); - fx.cx.module.define_data(data_id, &data_ctx).unwrap(); + // FIXME don't duplicate definitions in lazy jit mode + let _ = fx.cx.module.define_data(data_id, &data_ctx); data_id }