From b3a447a5658deaaf5567dddd027167db8112929c Mon Sep 17 00:00:00 2001 From: David Wood Date: Mon, 23 Sep 2024 18:46:23 +0100 Subject: [PATCH 1/6] mir_transform: implement forced inlining Adds `#[rustc_force_inline]` which is similar to always inlining but reports an error if the inlining was not possible, and which always attempts to inline annotated items, regardless of optimisation levels. It can only be applied to free functions to guarantee that the MIR inliner will be able to resolve calls. --- compiler/rustc_attr/src/builtin.rs | 16 + compiler/rustc_codegen_gcc/src/attributes.rs | 2 +- compiler/rustc_codegen_llvm/src/attributes.rs | 4 +- .../rustc_codegen_ssa/src/codegen_attrs.rs | 74 +- compiler/rustc_feature/src/builtin_attrs.rs | 4 + compiler/rustc_hir_typeck/src/coercion.rs | 23 +- compiler/rustc_lint_defs/src/builtin.rs | 29 + compiler/rustc_middle/src/mir/mono.rs | 11 +- compiler/rustc_middle/src/ty/error.rs | 3 + compiler/rustc_mir_transform/messages.ftl | 11 + .../src/cross_crate_inline.rs | 7 +- compiler/rustc_mir_transform/src/errors.rs | 28 +- compiler/rustc_mir_transform/src/inline.rs | 1492 ++++++++++------- compiler/rustc_mir_transform/src/lib.rs | 8 +- .../rustc_mir_transform/src/pass_manager.rs | 10 + compiler/rustc_mir_transform/src/validate.rs | 24 +- compiler/rustc_monomorphize/src/collector.rs | 8 + compiler/rustc_passes/messages.ftl | 8 + compiler/rustc_passes/src/check_attr.rs | 42 +- compiler/rustc_passes/src/errors.rs | 18 + compiler/rustc_span/src/symbol.rs | 1 + compiler/rustc_trait_selection/messages.ftl | 4 +- .../src/error_reporting/infer/mod.rs | 7 +- compiler/rustc_trait_selection/src/errors.rs | 11 +- compiler/rustc_type_ir/src/error.rs | 4 + .../inline/forced.caller.ForceInline.diff | 21 + tests/mir-opt/inline/forced.rs | 13 + .../forced_async.caller.ForceInline.diff | 12 + tests/mir-opt/inline/forced_async.rs | 14 + ...losure.caller-{closure#0}.ForceInline.diff | 21 + tests/mir-opt/inline/forced_closure.rs | 15 + tests/ui/force-inlining/auxiliary/callees.rs | 10 + tests/ui/force-inlining/cast.rs | 25 + tests/ui/force-inlining/cast.stderr | 40 + tests/ui/force-inlining/cross-crate.rs | 13 + tests/ui/force-inlining/deny-async.rs | 25 + tests/ui/force-inlining/deny-async.stderr | 19 + tests/ui/force-inlining/deny-closure.rs | 26 + tests/ui/force-inlining/deny-closure.stderr | 30 + tests/ui/force-inlining/deny.rs | 24 + tests/ui/force-inlining/deny.stderr | 19 + tests/ui/force-inlining/gate.rs | 12 + tests/ui/force-inlining/gate.stderr | 21 + tests/ui/force-inlining/invalid.rs | 164 ++ tests/ui/force-inlining/invalid.stderr | 378 +++++ 45 files changed, 2078 insertions(+), 673 deletions(-) create mode 100644 tests/mir-opt/inline/forced.caller.ForceInline.diff create mode 100644 tests/mir-opt/inline/forced.rs create mode 100644 tests/mir-opt/inline/forced_async.caller.ForceInline.diff create mode 100644 tests/mir-opt/inline/forced_async.rs create mode 100644 tests/mir-opt/inline/forced_closure.caller-{closure#0}.ForceInline.diff create mode 100644 tests/mir-opt/inline/forced_closure.rs create mode 100644 tests/ui/force-inlining/auxiliary/callees.rs create mode 100644 tests/ui/force-inlining/cast.rs create mode 100644 tests/ui/force-inlining/cast.stderr create mode 100644 tests/ui/force-inlining/cross-crate.rs create mode 100644 tests/ui/force-inlining/deny-async.rs create mode 100644 tests/ui/force-inlining/deny-async.stderr create mode 100644 tests/ui/force-inlining/deny-closure.rs create mode 100644 tests/ui/force-inlining/deny-closure.stderr create mode 100644 tests/ui/force-inlining/deny.rs create mode 100644 tests/ui/force-inlining/deny.stderr create mode 100644 tests/ui/force-inlining/gate.rs create mode 100644 tests/ui/force-inlining/gate.stderr create mode 100644 tests/ui/force-inlining/invalid.rs create mode 100644 tests/ui/force-inlining/invalid.stderr diff --git a/compiler/rustc_attr/src/builtin.rs b/compiler/rustc_attr/src/builtin.rs index 94f9727eb7fbe..2e4d192703c11 100644 --- a/compiler/rustc_attr/src/builtin.rs +++ b/compiler/rustc_attr/src/builtin.rs @@ -47,6 +47,22 @@ pub enum InlineAttr { Hint, Always, Never, + /// `#[rustc_force_inline]` forces inlining to happen in the MIR inliner - it reports an error + /// if the inlining cannot happen. It is limited to only free functions so that the calls + /// can always be resolved. + Force { + attr_span: Span, + reason: Option, + }, +} + +impl InlineAttr { + pub fn always(&self) -> bool { + match self { + InlineAttr::Always | InlineAttr::Force { .. } => true, + InlineAttr::None | InlineAttr::Hint | InlineAttr::Never => false, + } + } } #[derive(Clone, Encodable, Decodable, Debug, PartialEq, Eq, HashStable_Generic)] diff --git a/compiler/rustc_codegen_gcc/src/attributes.rs b/compiler/rustc_codegen_gcc/src/attributes.rs index d20e13e15b944..c106333dbf377 100644 --- a/compiler/rustc_codegen_gcc/src/attributes.rs +++ b/compiler/rustc_codegen_gcc/src/attributes.rs @@ -20,7 +20,7 @@ fn inline_attr<'gcc, 'tcx>( ) -> Option> { match inline { InlineAttr::Hint => Some(FnAttribute::Inline), - InlineAttr::Always => Some(FnAttribute::AlwaysInline), + InlineAttr::Always | InlineAttr::Force { .. } => Some(FnAttribute::AlwaysInline), InlineAttr::Never => { if cx.sess().target.arch != "amdgpu" { Some(FnAttribute::NoInline) diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs index 5552a2410601d..259d9a2080f55 100644 --- a/compiler/rustc_codegen_llvm/src/attributes.rs +++ b/compiler/rustc_codegen_llvm/src/attributes.rs @@ -37,7 +37,9 @@ fn inline_attr<'ll>(cx: &CodegenCx<'ll, '_>, inline: InlineAttr) -> Option<&'ll } match inline { InlineAttr::Hint => Some(AttributeKind::InlineHint.create_attr(cx.llcx)), - InlineAttr::Always => Some(AttributeKind::AlwaysInline.create_attr(cx.llcx)), + InlineAttr::Always | InlineAttr::Force { .. } => { + Some(AttributeKind::AlwaysInline.create_attr(cx.llcx)) + } InlineAttr::Never => { if cx.sess().target.arch != "amdgpu" { Some(AttributeKind::NoInline.create_attr(cx.llcx)) diff --git a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs index ab2d24e8d2dd7..3b14d47f4772c 100644 --- a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs +++ b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs @@ -18,6 +18,7 @@ use rustc_session::{Session, lint}; use rustc_span::symbol::Ident; use rustc_span::{Span, sym}; use rustc_target::spec::{SanitizerSet, abi}; +use tracing::debug; use crate::errors; use crate::target_features::{check_target_feature_trait_unsafe, from_target_feature_attr}; @@ -522,31 +523,50 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { mixed_export_name_no_mangle_lint_state.lint_if_mixed(tcx); codegen_fn_attrs.inline = attrs.iter().fold(InlineAttr::None, |ia, attr| { - if !attr.has_name(sym::inline) { - return ia; - } - match attr.meta_kind() { - Some(MetaItemKind::Word) => InlineAttr::Hint, - Some(MetaItemKind::List(ref items)) => { - inline_span = Some(attr.span); - if items.len() != 1 { - struct_span_code_err!(tcx.dcx(), attr.span, E0534, "expected one argument") - .emit(); - InlineAttr::None - } else if list_contains_name(items, sym::always) { - InlineAttr::Always - } else if list_contains_name(items, sym::never) { - InlineAttr::Never - } else { - struct_span_code_err!(tcx.dcx(), items[0].span(), E0535, "invalid argument") + if attr.has_name(sym::inline) { + match attr.meta_kind() { + Some(MetaItemKind::Word) => InlineAttr::Hint, + Some(MetaItemKind::List(ref items)) => { + inline_span = Some(attr.span); + if items.len() != 1 { + struct_span_code_err!(tcx.dcx(), attr.span, E0534, "expected one argument") + .emit(); + InlineAttr::None + } else if list_contains_name(items, sym::always) { + InlineAttr::Always + } else if list_contains_name(items, sym::never) { + InlineAttr::Never + } else { + struct_span_code_err!( + tcx.dcx(), + items[0].span(), + E0535, + "invalid argument" + ) .with_help("valid inline arguments are `always` and `never`") .emit(); - InlineAttr::None + InlineAttr::None + } } + Some(MetaItemKind::NameValue(_)) => ia, + None => ia, } - Some(MetaItemKind::NameValue(_)) => ia, - None => ia, + } else if attr.has_name(sym::rustc_force_inline) && tcx.features().rustc_attrs() { + match attr.meta_kind() { + Some(MetaItemKind::NameValue(lit)) => { + InlineAttr::Force { attr_span: attr.span, reason: Some(lit.symbol) } + } + Some(MetaItemKind::Word) => { + InlineAttr::Force { attr_span: attr.span, reason: None } + } + _ => { + debug!("`rustc_force_inline` not checked by attribute validation"); + ia + } + } + } else { + ia } }); @@ -601,7 +621,7 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { // is probably a poor usage of `#[inline(always)]` and easily avoided by not using the attribute. if tcx.features().target_feature_11() && tcx.is_closure_like(did.to_def_id()) - && codegen_fn_attrs.inline != InlineAttr::Always + && !codegen_fn_attrs.inline.always() { let owner_id = tcx.parent(did.to_def_id()); if tcx.def_kind(owner_id).has_codegen_attrs() { @@ -611,11 +631,15 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { } } - // If a function uses #[target_feature] it can't be inlined into general + // If a function uses `#[target_feature]` it can't be inlined into general // purpose functions as they wouldn't have the right target features - // enabled. For that reason we also forbid #[inline(always)] as it can't be + // enabled. For that reason we also forbid `#[inline(always)]` as it can't be // respected. - if !codegen_fn_attrs.target_features.is_empty() && codegen_fn_attrs.inline == InlineAttr::Always + // + // `#[rustc_force_inline]` doesn't need to be prohibited here, that + // is implemented entirely in rustc can attempt to inline and error if it cannot. + if !codegen_fn_attrs.target_features.is_empty() + && matches!(codegen_fn_attrs.inline, InlineAttr::Always) { if let Some(span) = inline_span { tcx.dcx().span_err( @@ -626,7 +650,7 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { } } - if !codegen_fn_attrs.no_sanitize.is_empty() && codegen_fn_attrs.inline == InlineAttr::Always { + if !codegen_fn_attrs.no_sanitize.is_empty() && codegen_fn_attrs.inline.always() { if let (Some(no_sanitize_span), Some(inline_span)) = (no_sanitize_span, inline_span) { let hir_id = tcx.local_def_id_to_hir_id(did); tcx.node_span_lint( diff --git a/compiler/rustc_feature/src/builtin_attrs.rs b/compiler/rustc_feature/src/builtin_attrs.rs index 3bf485c2eb6fb..33c1dc41d0838 100644 --- a/compiler/rustc_feature/src/builtin_attrs.rs +++ b/compiler/rustc_feature/src/builtin_attrs.rs @@ -1014,6 +1014,10 @@ pub const BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[ rustc_no_mir_inline, Normal, template!(Word), WarnFollowing, EncodeCrossCrate::Yes, "#[rustc_no_mir_inline] prevents the MIR inliner from inlining a function while not affecting codegen" ), + rustc_attr!( + rustc_force_inline, Normal, template!(Word, NameValueStr: "reason"), WarnFollowing, EncodeCrossCrate::Yes, + "#![rustc_force_inline] forces a free function to be inlined" + ), // ========================================================================== // Internal attributes, Testing: diff --git a/compiler/rustc_hir_typeck/src/coercion.rs b/compiler/rustc_hir_typeck/src/coercion.rs index bf41dcbe4a365..4044bf7778dfb 100644 --- a/compiler/rustc_hir_typeck/src/coercion.rs +++ b/compiler/rustc_hir_typeck/src/coercion.rs @@ -38,6 +38,7 @@ use std::ops::Deref; use rustc_abi::ExternAbi; +use rustc_attr::InlineAttr; use rustc_errors::codes::*; use rustc_errors::{Applicability, Diag, struct_span_code_err}; use rustc_hir as hir; @@ -923,11 +924,14 @@ impl<'f, 'tcx> Coerce<'f, 'tcx> { return Err(TypeError::IntrinsicCast); } - // Safe `#[target_feature]` functions are not assignable to safe fn pointers (RFC 2396). + let fn_attrs = self.tcx.codegen_fn_attrs(def_id); + if matches!(fn_attrs.inline, rustc_attr::InlineAttr::Force { .. }) { + return Err(TypeError::ForceInlineCast); + } - if b_hdr.safety == hir::Safety::Safe - && !self.tcx.codegen_fn_attrs(def_id).target_features.is_empty() - { + // Safe `#[target_feature]` functions are not assignable to safe fn pointers + // (RFC 2396). + if b_hdr.safety == hir::Safety::Safe && !fn_attrs.target_features.is_empty() { return Err(TypeError::TargetFeatureCast(def_id)); } } @@ -1194,6 +1198,17 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { return Ok(prev_ty); } + let is_force_inline = |ty: Ty<'tcx>| { + if let ty::FnDef(did, _) = ty.kind() { + matches!(self.tcx.codegen_fn_attrs(did).inline, InlineAttr::Force { .. }) + } else { + false + } + }; + if is_force_inline(prev_ty) || is_force_inline(new_ty) { + return Err(TypeError::ForceInlineCast); + } + // Special-case that coercion alone cannot handle: // Function items or non-capturing closures of differing IDs or GenericArgs. let (a_sig, b_sig) = { diff --git a/compiler/rustc_lint_defs/src/builtin.rs b/compiler/rustc_lint_defs/src/builtin.rs index 54e927df3c42b..e917a5d16d20b 100644 --- a/compiler/rustc_lint_defs/src/builtin.rs +++ b/compiler/rustc_lint_defs/src/builtin.rs @@ -46,6 +46,7 @@ declare_lint_pass! { EXPORTED_PRIVATE_DEPENDENCIES, FFI_UNWIND_CALLS, FORBIDDEN_LINT_GROUPS, + FORCED_INLINE, FUNCTION_ITEM_REFERENCES, FUZZY_PROVENANCE_CASTS, HIDDEN_GLOB_REEXPORTS, @@ -5178,3 +5179,31 @@ declare_lint! { reference: "issue #116558 ", }; } + +declare_lint! { + /// The `forced_inline` lint is emitted when a function annotated with + /// `#[rustc_force_inline]` was not able to be inlined. + /// + /// ### Example + /// + /// ```rust,ignore (needs rustc_attrs) + /// #[rustc_no_mir_inline] + /// #[rustc_force_inline] + /// fn foo() { } + /// + /// fn bar() { foo() } + /// ``` + /// + /// ### Explanation + /// + /// Functions can be marked as `#[rustc_force_inline]` in the standard + /// library if they are required to be inlined in order to uphold + /// security properties or some other similar guarantee. + /// + /// In some circumstances, these functions cannot be inlined and a + /// reason will be provided, this can either be rectified or the + /// lint can be silenced if the risk is acceptable. + pub FORCED_INLINE, + Deny, + "`#[rustc_force_inline]`-annotated function could not be inlined" +} diff --git a/compiler/rustc_middle/src/mir/mono.rs b/compiler/rustc_middle/src/mir/mono.rs index 1f50b67cb50d5..862df78a69f63 100644 --- a/compiler/rustc_middle/src/mir/mono.rs +++ b/compiler/rustc_middle/src/mir/mono.rs @@ -1,7 +1,6 @@ use std::fmt; use std::hash::Hash; -use rustc_attr::InlineAttr; use rustc_data_structures::base_n::{BaseNString, CASE_INSENSITIVE, ToBaseN}; use rustc_data_structures::fingerprint::Fingerprint; use rustc_data_structures::fx::FxIndexMap; @@ -114,7 +113,8 @@ impl<'tcx> MonoItem<'tcx> { return InstantiationMode::GloballyShared { may_conflict: false }; } - if let InlineAttr::Never = tcx.codegen_fn_attrs(instance.def_id()).inline + if let rustc_attr::InlineAttr::Never = + tcx.codegen_fn_attrs(instance.def_id()).inline && self.is_generic_fn() { // Upgrade inline(never) to a globally shared instance. @@ -133,9 +133,10 @@ impl<'tcx> MonoItem<'tcx> { // creating one copy of this `#[inline]` function which may // conflict with upstream crates as it could be an exported // symbol. - match tcx.codegen_fn_attrs(instance.def_id()).inline { - InlineAttr::Always => InstantiationMode::LocalCopy, - _ => InstantiationMode::GloballyShared { may_conflict: true }, + if tcx.codegen_fn_attrs(instance.def_id()).inline.always() { + InstantiationMode::LocalCopy + } else { + InstantiationMode::GloballyShared { may_conflict: true } } } MonoItem::Static(..) | MonoItem::GlobalAsm(..) => { diff --git a/compiler/rustc_middle/src/ty/error.rs b/compiler/rustc_middle/src/ty/error.rs index 4a82af3255950..2c32ada1ec1cb 100644 --- a/compiler/rustc_middle/src/ty/error.rs +++ b/compiler/rustc_middle/src/ty/error.rs @@ -109,6 +109,9 @@ impl<'tcx> TypeError<'tcx> { TypeError::ConstMismatch(ref values) => { format!("expected `{}`, found `{}`", values.expected, values.found).into() } + TypeError::ForceInlineCast => { + "cannot coerce functions which must be inlined to function pointers".into() + } TypeError::IntrinsicCast => "cannot coerce intrinsics to function pointers".into(), TypeError::TargetFeatureCast(_) => { "cannot coerce functions with `#[target_feature]` to safe function pointers".into() diff --git a/compiler/rustc_mir_transform/messages.ftl b/compiler/rustc_mir_transform/messages.ftl index d00bfc66a6a57..b0c023cca8242 100644 --- a/compiler/rustc_mir_transform/messages.ftl +++ b/compiler/rustc_mir_transform/messages.ftl @@ -19,6 +19,17 @@ mir_transform_ffi_unwind_call = call to {$foreign -> mir_transform_fn_item_ref = taking a reference to a function item does not give a function pointer .suggestion = cast `{$ident}` to obtain a function pointer +mir_transform_force_inline = + `{$callee}` could not be inlined into `{$caller}` but is required to be inlined + .call = ...`{$callee}` called here + .attr = inlining due to this annotation + .caller = within `{$caller}`... + .callee = `{$callee}` defined here + .note = could not be inlined due to: {$reason} + +mir_transform_force_inline_justification = + `{$callee}` is required to be inlined to: {$sym} + mir_transform_must_not_suspend = {$pre}`{$def_path}`{$post} held across a suspend point, but should not be .label = the value is held across this suspend point .note = {$reason} diff --git a/compiler/rustc_mir_transform/src/cross_crate_inline.rs b/compiler/rustc_mir_transform/src/cross_crate_inline.rs index 589be81558c2b..d0ec106cfc721 100644 --- a/compiler/rustc_mir_transform/src/cross_crate_inline.rs +++ b/compiler/rustc_mir_transform/src/cross_crate_inline.rs @@ -46,7 +46,7 @@ fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool { // #[inline(never)] to force code generation. match codegen_fn_attrs.inline { InlineAttr::Never => return false, - InlineAttr::Hint | InlineAttr::Always => return true, + InlineAttr::Hint | InlineAttr::Always | InlineAttr::Force { .. } => return true, _ => {} } @@ -69,8 +69,9 @@ fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool { // Don't do any inference if codegen optimizations are disabled and also MIR inlining is not // enabled. This ensures that we do inference even if someone only passes -Zinline-mir, // which is less confusing than having to also enable -Copt-level=1. - if matches!(tcx.sess.opts.optimize, OptLevel::No) && !pm::should_run_pass(tcx, &inline::Inline) - { + let inliner_will_run = pm::should_run_pass(tcx, &inline::Inline) + || inline::ForceInline::should_run_pass_for_callee(tcx, def_id.to_def_id()); + if matches!(tcx.sess.opts.optimize, OptLevel::No) && !inliner_will_run { return false; } diff --git a/compiler/rustc_mir_transform/src/errors.rs b/compiler/rustc_mir_transform/src/errors.rs index 2d9eeddea2e22..015633d145f15 100644 --- a/compiler/rustc_mir_transform/src/errors.rs +++ b/compiler/rustc_mir_transform/src/errors.rs @@ -4,8 +4,8 @@ use rustc_macros::{Diagnostic, LintDiagnostic, Subdiagnostic}; use rustc_middle::mir::AssertKind; use rustc_middle::ty::TyCtxt; use rustc_session::lint::{self, Lint}; -use rustc_span::Span; use rustc_span::def_id::DefId; +use rustc_span::{Span, Symbol}; use crate::fluent_generated as fluent; @@ -142,3 +142,29 @@ pub(crate) struct MustNotSuspendReason { #[note(mir_transform_note2)] #[help] pub(crate) struct UndefinedTransmute; + +#[derive(Diagnostic)] +#[diag(mir_transform_force_inline)] +#[note] +pub(crate) struct ForceInlineFailure { + #[label(mir_transform_caller)] + pub caller_span: Span, + #[label(mir_transform_callee)] + pub callee_span: Span, + #[label(mir_transform_attr)] + pub attr_span: Span, + #[primary_span] + #[label(mir_transform_call)] + pub call_span: Span, + pub callee: String, + pub caller: String, + pub reason: &'static str, + #[subdiagnostic] + pub justification: Option, +} + +#[derive(Subdiagnostic)] +#[note(mir_transform_force_inline_justification)] +pub(crate) struct ForceInlineJustification { + pub sym: Symbol, +} diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs index 79c38b50459c4..424e774b3a1c7 100644 --- a/compiler/rustc_mir_transform/src/inline.rs +++ b/compiler/rustc_mir_transform/src/inline.rs @@ -6,7 +6,7 @@ use std::ops::{Range, RangeFrom}; use rustc_abi::{ExternAbi, FieldIdx}; use rustc_attr::InlineAttr; use rustc_hir::def::DefKind; -use rustc_hir::def_id::DefId; +use rustc_hir::def_id::{DefId, LocalDefId}; use rustc_index::Idx; use rustc_index::bit_set::BitSet; use rustc_middle::bug; @@ -29,10 +29,6 @@ pub(crate) mod cycle; const TOP_DOWN_DEPTH_LIMIT: usize = 5; -// Made public so that `mir_drops_elaborated_and_const_checked` can be overridden -// by custom rustc drivers, running all the steps by themselves. See #114628. -pub struct Inline; - #[derive(Clone, Debug)] struct CallSite<'tcx> { callee: Instance<'tcx>, @@ -41,14 +37,12 @@ struct CallSite<'tcx> { source_info: SourceInfo, } +// Made public so that `mir_drops_elaborated_and_const_checked` can be overridden +// by custom rustc drivers, running all the steps by themselves. See #114628. +pub struct Inline; + impl<'tcx> crate::MirPass<'tcx> for Inline { fn is_enabled(&self, sess: &rustc_session::Session) -> bool { - // FIXME(#127234): Coverage instrumentation currently doesn't handle inlined - // MIR correctly when Modified Condition/Decision Coverage is enabled. - if sess.instrument_coverage_mcdc() { - return false; - } - if let Some(enabled) = sess.opts.unstable_opts.inline_mir { return enabled; } @@ -67,7 +61,7 @@ impl<'tcx> crate::MirPass<'tcx> for Inline { fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { let span = trace_span!("inline", body = %tcx.def_path_str(body.source.def_id())); let _guard = span.enter(); - if inline(tcx, body) { + if inline::>(tcx, body) { debug!("running simplify cfg on {:?}", body.source); simplify_cfg(body); deref_finder(tcx, body); @@ -75,47 +69,84 @@ impl<'tcx> crate::MirPass<'tcx> for Inline { } } -fn inline<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> bool { - let def_id = body.source.def_id().expect_local(); +pub struct ForceInline; - // Only do inlining into fn bodies. - if !tcx.hir().body_owner_kind(def_id).is_fn_or_closure() { - return false; +impl ForceInline { + pub fn should_run_pass_for_callee<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> bool { + matches!(tcx.codegen_fn_attrs(def_id).inline, InlineAttr::Force { .. }) } - if body.source.promoted.is_some() { - return false; +} + +impl<'tcx> crate::MirPass<'tcx> for ForceInline { + fn is_enabled(&self, _: &rustc_session::Session) -> bool { + true } - // Avoid inlining into coroutines, since their `optimized_mir` is used for layout computation, - // which can create a cycle, even when no attempt is made to inline the function in the other - // direction. - if body.coroutine.is_some() { - return false; + + fn can_be_overridden(&self) -> bool { + false } - let typing_env = body.typing_env(tcx); - let codegen_fn_attrs = tcx.codegen_fn_attrs(def_id); + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + let span = trace_span!("force_inline", body = %tcx.def_path_str(body.source.def_id())); + let _guard = span.enter(); + if inline::>(tcx, body) { + debug!("running simplify cfg on {:?}", body.source); + simplify_cfg(body); + deref_finder(tcx, body); + } + } +} - let mut this = Inliner { - tcx, - typing_env, - codegen_fn_attrs, - history: Vec::new(), - changed: false, - caller_is_inline_forwarder: matches!( - codegen_fn_attrs.inline, - InlineAttr::Hint | InlineAttr::Always - ) && body_is_forwarder(body), - }; - let blocks = START_BLOCK..body.basic_blocks.next_index(); - this.process_blocks(body, blocks); - this.changed +trait Inliner<'tcx> { + fn new(tcx: TyCtxt<'tcx>, def_id: LocalDefId, body: &Body<'tcx>) -> Self; + + fn tcx(&self) -> TyCtxt<'tcx>; + fn typing_env(&self) -> ty::TypingEnv<'tcx>; + fn history(&self) -> &[DefId]; + fn caller_def_id(&self) -> LocalDefId; + + /// Has the caller body been changed? + fn changed(self) -> bool; + + /// Should inlining happen for a given callee? + fn should_inline_for_callee(&self, def_id: DefId) -> bool; + + fn check_caller_mir_body(&self, body: &Body<'tcx>) -> bool; + + /// Returns inlining decision that is based on the examination of callee MIR body. + /// Assumes that codegen attributes have been checked for compatibility already. + fn check_callee_mir_body( + &self, + callsite: &CallSite<'tcx>, + callee_body: &Body<'tcx>, + callee_attrs: &CodegenFnAttrs, + cross_crate_inlinable: bool, + ) -> Result<(), &'static str>; + + // How many callsites in a body are we allowed to inline? We need to limit this in order + // to prevent super-linear growth in MIR size. + fn inline_limit_for_block(&self) -> Option; + + /// Called when inlining succeeds. + fn on_inline_success( + &mut self, + callsite: &CallSite<'tcx>, + caller_body: &mut Body<'tcx>, + new_blocks: std::ops::Range, + ); + + /// Called when inlining failed or was not performed. + fn on_inline_failure(&self, callsite: &CallSite<'tcx>, reason: &'static str); + + /// Called when the inline limit for a body is reached. + fn on_inline_limit_reached(&self) -> bool; } -struct Inliner<'tcx> { +struct ForceInliner<'tcx> { tcx: TyCtxt<'tcx>, typing_env: ty::TypingEnv<'tcx>, - /// Caller codegen attributes. - codegen_fn_attrs: &'tcx CodegenFnAttrs, + /// `DefId` of caller. + def_id: LocalDefId, /// Stack of inlined instances. /// We only check the `DefId` and not the args because we want to /// avoid inlining cases of polymorphic recursion. @@ -124,365 +155,190 @@ struct Inliner<'tcx> { history: Vec, /// Indicates that the caller body has been modified. changed: bool, - /// Indicates that the caller is #[inline] and just calls another function, - /// and thus we can inline less into it as it'll be inlined itself. - caller_is_inline_forwarder: bool, } -impl<'tcx> Inliner<'tcx> { - fn process_blocks(&mut self, caller_body: &mut Body<'tcx>, blocks: Range) { - // How many callsites in this body are we allowed to inline? We need to limit this in order - // to prevent super-linear growth in MIR size - let inline_limit = match self.history.len() { - 0 => usize::MAX, - 1..=TOP_DOWN_DEPTH_LIMIT => 1, - _ => return, - }; - let mut inlined_count = 0; - for bb in blocks { - let bb_data = &caller_body[bb]; - if bb_data.is_cleanup { - continue; - } - - let Some(callsite) = self.resolve_callsite(caller_body, bb, bb_data) else { - continue; - }; - - let span = trace_span!("process_blocks", %callsite.callee, ?bb); - let _guard = span.enter(); - - match self.try_inlining(caller_body, &callsite) { - Err(reason) => { - debug!("not-inlined {} [{}]", callsite.callee, reason); - } - Ok(new_blocks) => { - debug!("inlined {}", callsite.callee); - self.changed = true; - - self.history.push(callsite.callee.def_id()); - self.process_blocks(caller_body, new_blocks); - self.history.pop(); - - inlined_count += 1; - if inlined_count == inline_limit { - debug!("inline count reached"); - return; - } - } - } - } +impl<'tcx> Inliner<'tcx> for ForceInliner<'tcx> { + fn new(tcx: TyCtxt<'tcx>, def_id: LocalDefId, body: &Body<'tcx>) -> Self { + Self { tcx, typing_env: body.typing_env(tcx), def_id, history: Vec::new(), changed: false } } - /// Attempts to inline a callsite into the caller body. When successful returns basic blocks - /// containing the inlined body. Otherwise returns an error describing why inlining didn't take - /// place. - fn try_inlining( - &self, - caller_body: &mut Body<'tcx>, - callsite: &CallSite<'tcx>, - ) -> Result, &'static str> { - self.check_mir_is_available(caller_body, callsite.callee)?; - - let callee_attrs = self.tcx.codegen_fn_attrs(callsite.callee.def_id()); - let cross_crate_inlinable = self.tcx.cross_crate_inlinable(callsite.callee.def_id()); - self.check_codegen_attributes(callsite, callee_attrs, cross_crate_inlinable)?; - - // Intrinsic fallback bodies are automatically made cross-crate inlineable, - // but at this stage we don't know whether codegen knows the intrinsic, - // so just conservatively don't inline it. - if self.tcx.has_attr(callsite.callee.def_id(), sym::rustc_intrinsic) { - return Err("Callee is an intrinsic, do not inline fallback bodies"); - } - - let terminator = caller_body[callsite.block].terminator.as_ref().unwrap(); - let TerminatorKind::Call { args, destination, .. } = &terminator.kind else { bug!() }; - let destination_ty = destination.ty(&caller_body.local_decls, self.tcx).ty; - for arg in args { - if !arg.node.ty(&caller_body.local_decls, self.tcx).is_sized(self.tcx, self.typing_env) - { - // We do not allow inlining functions with unsized params. Inlining these functions - // could create unsized locals, which are unsound and being phased out. - return Err("Call has unsized argument"); - } - } - - let callee_body = try_instance_mir(self.tcx, callsite.callee.def)?; - self.check_mir_body(callsite, callee_body, callee_attrs, cross_crate_inlinable)?; - - let Ok(callee_body) = callsite.callee.try_instantiate_mir_and_normalize_erasing_regions( - self.tcx, - self.typing_env, - ty::EarlyBinder::bind(callee_body.clone()), - ) else { - return Err("failed to normalize callee body"); - }; - - // Normally, this shouldn't be required, but trait normalization failure can create a - // validation ICE. - if !validate_types(self.tcx, self.typing_env, &callee_body, &caller_body).is_empty() { - return Err("failed to validate callee body"); - } - - // Check call signature compatibility. - // Normally, this shouldn't be required, but trait normalization failure can create a - // validation ICE. - let output_type = callee_body.return_ty(); - if !util::sub_types(self.tcx, self.typing_env, output_type, destination_ty) { - trace!(?output_type, ?destination_ty); - return Err("failed to normalize return type"); - } - if callsite.fn_sig.abi() == ExternAbi::RustCall { - // FIXME: Don't inline user-written `extern "rust-call"` functions, - // since this is generally perf-negative on rustc, and we hope that - // LLVM will inline these functions instead. - if callee_body.spread_arg.is_some() { - return Err("do not inline user-written rust-call functions"); - } + fn tcx(&self) -> TyCtxt<'tcx> { + self.tcx + } - let (self_arg, arg_tuple) = match &args[..] { - [arg_tuple] => (None, arg_tuple), - [self_arg, arg_tuple] => (Some(self_arg), arg_tuple), - _ => bug!("Expected `rust-call` to have 1 or 2 args"), - }; + fn typing_env(&self) -> ty::TypingEnv<'tcx> { + self.typing_env + } - let self_arg_ty = - self_arg.map(|self_arg| self_arg.node.ty(&caller_body.local_decls, self.tcx)); + fn history(&self) -> &[DefId] { + &self.history + } - let arg_tuple_ty = arg_tuple.node.ty(&caller_body.local_decls, self.tcx); - let ty::Tuple(arg_tuple_tys) = *arg_tuple_ty.kind() else { - bug!("Closure arguments are not passed as a tuple"); - }; + fn caller_def_id(&self) -> LocalDefId { + self.def_id + } - for (arg_ty, input) in - self_arg_ty.into_iter().chain(arg_tuple_tys).zip(callee_body.args_iter()) - { - let input_type = callee_body.local_decls[input].ty; - if !util::sub_types(self.tcx, self.typing_env, input_type, arg_ty) { - trace!(?arg_ty, ?input_type); - return Err("failed to normalize tuple argument type"); - } - } - } else { - for (arg, input) in args.iter().zip(callee_body.args_iter()) { - let input_type = callee_body.local_decls[input].ty; - let arg_ty = arg.node.ty(&caller_body.local_decls, self.tcx); - if !util::sub_types(self.tcx, self.typing_env, input_type, arg_ty) { - trace!(?arg_ty, ?input_type); - return Err("failed to normalize argument type"); - } - } - } + fn changed(self) -> bool { + self.changed + } - let old_blocks = caller_body.basic_blocks.next_index(); - self.inline_call(caller_body, callsite, callee_body); - let new_blocks = old_blocks..caller_body.basic_blocks.next_index(); + fn should_inline_for_callee(&self, def_id: DefId) -> bool { + ForceInline::should_run_pass_for_callee(self.tcx(), def_id) + } - Ok(new_blocks) + fn check_caller_mir_body(&self, _: &Body<'tcx>) -> bool { + true } - fn check_mir_is_available( + #[instrument(level = "debug", skip(self, callee_body))] + fn check_callee_mir_body( &self, - caller_body: &Body<'tcx>, - callee: Instance<'tcx>, + _: &CallSite<'tcx>, + callee_body: &Body<'tcx>, + _: &CodegenFnAttrs, + _: bool, ) -> Result<(), &'static str> { - let caller_def_id = caller_body.source.def_id(); - let callee_def_id = callee.def_id(); - if callee_def_id == caller_def_id { - return Err("self-recursion"); - } - - match callee.def { - InstanceKind::Item(_) => { - // If there is no MIR available (either because it was not in metadata or - // because it has no MIR because it's an extern function), then the inliner - // won't cause cycles on this. - if !self.tcx.is_mir_available(callee_def_id) { - return Err("item MIR unavailable"); - } - } - // These have no own callable MIR. - InstanceKind::Intrinsic(_) | InstanceKind::Virtual(..) => { - return Err("instance without MIR (intrinsic / virtual)"); - } - - // FIXME(#127030): `ConstParamHasTy` has bad interactions with - // the drop shim builder, which does not evaluate predicates in - // the correct param-env for types being dropped. Stall resolving - // the MIR for this instance until all of its const params are - // substituted. - InstanceKind::DropGlue(_, Some(ty)) if ty.has_type_flags(TypeFlags::HAS_CT_PARAM) => { - return Err("still needs substitution"); - } - - // This cannot result in an immediate cycle since the callee MIR is a shim, which does - // not get any optimizations run on it. Any subsequent inlining may cause cycles, but we - // do not need to catch this here, we can wait until the inliner decides to continue - // inlining a second time. - InstanceKind::VTableShim(_) - | InstanceKind::ReifyShim(..) - | InstanceKind::FnPtrShim(..) - | InstanceKind::ClosureOnceShim { .. } - | InstanceKind::ConstructCoroutineInClosureShim { .. } - | InstanceKind::DropGlue(..) - | InstanceKind::CloneShim(..) - | InstanceKind::ThreadLocalShim(..) - | InstanceKind::FnPtrAddrShim(..) - | InstanceKind::AsyncDropGlueCtorShim(..) => return Ok(()), - } + if let Some(_) = callee_body.tainted_by_errors { Err("body has errors") } else { Ok(()) } + } - if self.tcx.is_constructor(callee_def_id) { - trace!("constructors always have MIR"); - // Constructor functions cannot cause a query cycle. - return Ok(()); - } + fn inline_limit_for_block(&self) -> Option { + Some(usize::MAX) + } - if callee_def_id.is_local() { - // If we know for sure that the function we're calling will itself try to - // call us, then we avoid inlining that function. - if self.tcx.mir_callgraph_reachable((callee, caller_def_id.expect_local())) { - return Err("caller might be reachable from callee (query cycle avoidance)"); - } + fn on_inline_success( + &mut self, + callsite: &CallSite<'tcx>, + caller_body: &mut Body<'tcx>, + new_blocks: std::ops::Range, + ) { + self.changed = true; - Ok(()) - } else { - // This cannot result in an immediate cycle since the callee MIR is from another crate - // and is already optimized. Any subsequent inlining may cause cycles, but we do - // not need to catch this here, we can wait until the inliner decides to continue - // inlining a second time. - trace!("functions from other crates always have MIR"); - Ok(()) - } + self.history.push(callsite.callee.def_id()); + process_blocks(self, caller_body, new_blocks); + self.history.pop(); } - fn resolve_callsite( - &self, - caller_body: &Body<'tcx>, - bb: BasicBlock, - bb_data: &BasicBlockData<'tcx>, - ) -> Option> { - // Only consider direct calls to functions - let terminator = bb_data.terminator(); - - // FIXME(explicit_tail_calls): figure out if we can inline tail calls - if let TerminatorKind::Call { ref func, fn_span, .. } = terminator.kind { - let func_ty = func.ty(caller_body, self.tcx); - if let ty::FnDef(def_id, args) = *func_ty.kind() { - // To resolve an instance its args have to be fully normalized. - let args = self.tcx.try_normalize_erasing_regions(self.typing_env, args).ok()?; - let callee = Instance::try_resolve(self.tcx, self.typing_env, def_id, args) - .ok() - .flatten()?; - - if let InstanceKind::Virtual(..) | InstanceKind::Intrinsic(_) = callee.def { - return None; - } - - if self.history.contains(&callee.def_id()) { - return None; - } + fn on_inline_failure(&self, callsite: &CallSite<'tcx>, reason: &'static str) { + let tcx = self.tcx(); + let InlineAttr::Force { attr_span, reason: justification } = + tcx.codegen_fn_attrs(callsite.callee.def_id()).inline + else { + bug!("called on item without required inlining"); + }; - let fn_sig = self.tcx.fn_sig(def_id).instantiate(self.tcx, args); + let call_span = callsite.source_info.span; + tcx.dcx().emit_err(crate::errors::ForceInlineFailure { + call_span, + attr_span, + caller_span: tcx.def_span(self.def_id), + caller: tcx.def_path_str(self.def_id), + callee_span: tcx.def_span(callsite.callee.def_id()), + callee: tcx.def_path_str(callsite.callee.def_id()), + reason, + justification: justification.map(|sym| crate::errors::ForceInlineJustification { sym }), + }); + } - // Additionally, check that the body that we're inlining actually agrees - // with the ABI of the trait that the item comes from. - if let InstanceKind::Item(instance_def_id) = callee.def - && self.tcx.def_kind(instance_def_id) == DefKind::AssocFn - && let instance_fn_sig = self.tcx.fn_sig(instance_def_id).skip_binder() - && instance_fn_sig.abi() != fn_sig.abi() - { - return None; - } + fn on_inline_limit_reached(&self) -> bool { + false + } +} - let source_info = SourceInfo { span: fn_span, ..terminator.source_info }; +struct NormalInliner<'tcx> { + tcx: TyCtxt<'tcx>, + typing_env: ty::TypingEnv<'tcx>, + /// `DefId` of caller. + def_id: LocalDefId, + /// Stack of inlined instances. + /// We only check the `DefId` and not the args because we want to + /// avoid inlining cases of polymorphic recursion. + /// The number of `DefId`s is finite, so checking history is enough + /// to ensure that we do not loop endlessly while inlining. + history: Vec, + /// Indicates that the caller body has been modified. + changed: bool, + /// Indicates that the caller is #[inline] and just calls another function, + /// and thus we can inline less into it as it'll be inlined itself. + caller_is_inline_forwarder: bool, +} - return Some(CallSite { callee, fn_sig, block: bb, source_info }); - } +impl<'tcx> Inliner<'tcx> for NormalInliner<'tcx> { + fn new(tcx: TyCtxt<'tcx>, def_id: LocalDefId, body: &Body<'tcx>) -> Self { + let typing_env = body.typing_env(tcx); + let codegen_fn_attrs = tcx.codegen_fn_attrs(def_id); + + Self { + tcx, + typing_env, + def_id, + history: Vec::new(), + changed: false, + caller_is_inline_forwarder: matches!( + codegen_fn_attrs.inline, + InlineAttr::Hint | InlineAttr::Always | InlineAttr::Force { .. } + ) && body_is_forwarder(body), } - - None } - /// Returns an error if inlining is not possible based on codegen attributes alone. A success - /// indicates that inlining decision should be based on other criteria. - fn check_codegen_attributes( - &self, - callsite: &CallSite<'tcx>, - callee_attrs: &CodegenFnAttrs, - cross_crate_inlinable: bool, - ) -> Result<(), &'static str> { - if self.tcx.has_attr(callsite.callee.def_id(), sym::rustc_no_mir_inline) { - return Err("#[rustc_no_mir_inline]"); - } + fn tcx(&self) -> TyCtxt<'tcx> { + self.tcx + } - if let InlineAttr::Never = callee_attrs.inline { - return Err("never inline hint"); - } + fn caller_def_id(&self) -> LocalDefId { + self.def_id + } - // Reachability pass defines which functions are eligible for inlining. Generally inlining - // other functions is incorrect because they could reference symbols that aren't exported. - let is_generic = callsite.callee.args.non_erasable_generics().next().is_some(); - if !is_generic && !cross_crate_inlinable { - return Err("not exported"); - } + fn typing_env(&self) -> ty::TypingEnv<'tcx> { + self.typing_env + } - if callsite.fn_sig.c_variadic() { - return Err("C variadic"); - } + fn history(&self) -> &[DefId] { + &self.history + } - if callee_attrs.flags.contains(CodegenFnAttrFlags::COLD) { - return Err("cold"); - } + fn changed(self) -> bool { + self.changed + } - if callee_attrs.no_sanitize != self.codegen_fn_attrs.no_sanitize { - return Err("incompatible sanitizer set"); - } + fn should_inline_for_callee(&self, _: DefId) -> bool { + true + } - // Two functions are compatible if the callee has no attribute (meaning - // that it's codegen agnostic), or sets an attribute that is identical - // to this function's attribute. - if callee_attrs.instruction_set.is_some() - && callee_attrs.instruction_set != self.codegen_fn_attrs.instruction_set - { - return Err("incompatible instruction set"); + fn check_caller_mir_body(&self, body: &Body<'tcx>) -> bool { + if body.source.promoted.is_some() { + return false; } - let callee_feature_names = callee_attrs.target_features.iter().map(|f| f.name); - let this_feature_names = self.codegen_fn_attrs.target_features.iter().map(|f| f.name); - if callee_feature_names.ne(this_feature_names) { - // In general it is not correct to inline a callee with target features that are a - // subset of the caller. This is because the callee might contain calls, and the ABI of - // those calls depends on the target features of the surrounding function. By moving a - // `Call` terminator from one MIR body to another with more target features, we might - // change the ABI of that call! - return Err("incompatible target features"); + // Avoid inlining into coroutines, since their `optimized_mir` is used for layout computation, + // which can create a cycle, even when no attempt is made to inline the function in the other + // direction. + if body.coroutine.is_some() { + return false; } - Ok(()) + true } - /// Returns inlining decision that is based on the examination of callee MIR body. - /// Assumes that codegen attributes have been checked for compatibility already. #[instrument(level = "debug", skip(self, callee_body))] - fn check_mir_body( + fn check_callee_mir_body( &self, callsite: &CallSite<'tcx>, callee_body: &Body<'tcx>, callee_attrs: &CodegenFnAttrs, cross_crate_inlinable: bool, ) -> Result<(), &'static str> { - let tcx = self.tcx; + let tcx = self.tcx(); if let Some(_) = callee_body.tainted_by_errors { - return Err("Body is tainted"); + return Err("body has errors"); } let mut threshold = if self.caller_is_inline_forwarder { - self.tcx.sess.opts.unstable_opts.inline_mir_forwarder_threshold.unwrap_or(30) + tcx.sess.opts.unstable_opts.inline_mir_forwarder_threshold.unwrap_or(30) } else if cross_crate_inlinable { - self.tcx.sess.opts.unstable_opts.inline_mir_hint_threshold.unwrap_or(100) + tcx.sess.opts.unstable_opts.inline_mir_hint_threshold.unwrap_or(100) } else { - self.tcx.sess.opts.unstable_opts.inline_mir_threshold.unwrap_or(50) + tcx.sess.opts.unstable_opts.inline_mir_threshold.unwrap_or(50) }; // Give a bonus functions with a small number of blocks, @@ -496,7 +352,7 @@ impl<'tcx> Inliner<'tcx> { // FIXME: Give a bonus to functions with only a single caller let mut checker = - CostChecker::new(self.tcx, self.typing_env, Some(callsite.callee), callee_body); + CostChecker::new(tcx, self.typing_env(), Some(callsite.callee), callee_body); checker.add_function_level_costs(); @@ -512,20 +368,20 @@ impl<'tcx> Inliner<'tcx> { checker.visit_basic_block_data(bb, blk); let term = blk.terminator(); + let caller_attrs = tcx.codegen_fn_attrs(self.caller_def_id()); if let TerminatorKind::Drop { ref place, target, unwind, replace: _ } = term.kind { work_list.push(target); // If the place doesn't actually need dropping, treat it like a regular goto. - let ty = callsite.callee.instantiate_mir( - self.tcx, - ty::EarlyBinder::bind(&place.ty(callee_body, tcx).ty), - ); - if ty.needs_drop(tcx, self.typing_env) + let ty = callsite + .callee + .instantiate_mir(tcx, ty::EarlyBinder::bind(&place.ty(callee_body, tcx).ty)); + if ty.needs_drop(tcx, self.typing_env()) && let UnwindAction::Cleanup(unwind) = unwind { work_list.push(unwind); } - } else if callee_attrs.instruction_set != self.codegen_fn_attrs.instruction_set + } else if callee_attrs.instruction_set != caller_attrs.instruction_set && matches!(term.kind, TerminatorKind::InlineAsm { .. }) { // During the attribute checking stage we allow a callee with no @@ -533,7 +389,7 @@ impl<'tcx> Inliner<'tcx> { // assign one. However, during this stage we require an exact match when any // inline-asm is detected. LLVM will still possibly do an inline later on // if the no-attribute function ends up with the same instruction set anyway. - return Err("Cannot move inline-asm across instruction sets"); + return Err("cannot move inline-asm across instruction sets"); } else if let TerminatorKind::TailCall { .. } = term.kind { // FIXME(explicit_tail_calls): figure out how exactly functions containing tail // calls can be inlined (and if they even should) @@ -557,319 +413,707 @@ impl<'tcx> Inliner<'tcx> { } } - fn inline_call( - &self, - caller_body: &mut Body<'tcx>, + fn inline_limit_for_block(&self) -> Option { + match self.history.len() { + 0 => Some(usize::MAX), + 1..=TOP_DOWN_DEPTH_LIMIT => Some(1), + _ => None, + } + } + + fn on_inline_success( + &mut self, callsite: &CallSite<'tcx>, - mut callee_body: Body<'tcx>, + caller_body: &mut Body<'tcx>, + new_blocks: std::ops::Range, ) { - let terminator = caller_body[callsite.block].terminator.take().unwrap(); - let TerminatorKind::Call { func, args, destination, unwind, target, .. } = terminator.kind - else { - bug!("unexpected terminator kind {:?}", terminator.kind); - }; + self.changed = true; - let return_block = if let Some(block) = target { - // Prepare a new block for code that should execute when call returns. We don't use - // target block directly since it might have other predecessors. - let mut data = BasicBlockData::new(Some(Terminator { - source_info: terminator.source_info, - kind: TerminatorKind::Goto { target: block }, - })); - data.is_cleanup = caller_body[block].is_cleanup; - Some(caller_body.basic_blocks_mut().push(data)) - } else { - None + self.history.push(callsite.callee.def_id()); + process_blocks(self, caller_body, new_blocks); + self.history.pop(); + } + + fn on_inline_limit_reached(&self) -> bool { + true + } + + fn on_inline_failure(&self, _: &CallSite<'tcx>, _: &'static str) {} +} + +fn inline<'tcx, T: Inliner<'tcx>>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> bool { + let def_id = body.source.def_id().expect_local(); + + // Only do inlining into fn bodies. + if !tcx.hir().body_owner_kind(def_id).is_fn_or_closure() { + return false; + } + + let mut inliner = T::new(tcx, def_id, body); + if !inliner.check_caller_mir_body(body) { + return false; + } + + let blocks = START_BLOCK..body.basic_blocks.next_index(); + process_blocks(&mut inliner, body, blocks); + inliner.changed() +} + +fn process_blocks<'tcx, I: Inliner<'tcx>>( + inliner: &mut I, + caller_body: &mut Body<'tcx>, + blocks: Range, +) { + let Some(inline_limit) = inliner.inline_limit_for_block() else { return }; + let mut inlined_count = 0; + for bb in blocks { + let bb_data = &caller_body[bb]; + if bb_data.is_cleanup { + continue; + } + + let Some(callsite) = resolve_callsite(inliner, caller_body, bb, bb_data) else { + continue; }; - // If the call is something like `a[*i] = f(i)`, where - // `i : &mut usize`, then just duplicating the `a[*i]` - // Place could result in two different locations if `f` - // writes to `i`. To prevent this we need to create a temporary - // borrow of the place and pass the destination as `*temp` instead. - fn dest_needs_borrow(place: Place<'_>) -> bool { - for elem in place.projection.iter() { - match elem { - ProjectionElem::Deref | ProjectionElem::Index(_) => return true, - _ => {} + let span = trace_span!("process_blocks", %callsite.callee, ?bb); + let _guard = span.enter(); + + if !inliner.should_inline_for_callee(callsite.callee.def_id()) { + debug!("not enabled"); + continue; + } + + match try_inlining(inliner, caller_body, &callsite) { + Err(reason) => { + debug!("not-inlined {} [{}]", callsite.callee, reason); + inliner.on_inline_failure(&callsite, reason); + } + Ok(new_blocks) => { + debug!("inlined {}", callsite.callee); + inliner.on_inline_success(&callsite, caller_body, new_blocks); + + inlined_count += 1; + if inlined_count == inline_limit { + if inliner.on_inline_limit_reached() { + return; + } } } + } + } +} + +fn resolve_callsite<'tcx, I: Inliner<'tcx>>( + inliner: &I, + caller_body: &Body<'tcx>, + bb: BasicBlock, + bb_data: &BasicBlockData<'tcx>, +) -> Option> { + let tcx = inliner.tcx(); + // Only consider direct calls to functions + let terminator = bb_data.terminator(); + + // FIXME(explicit_tail_calls): figure out if we can inline tail calls + if let TerminatorKind::Call { ref func, fn_span, .. } = terminator.kind { + let func_ty = func.ty(caller_body, tcx); + if let ty::FnDef(def_id, args) = *func_ty.kind() { + // To resolve an instance its args have to be fully normalized. + let args = tcx.try_normalize_erasing_regions(inliner.typing_env(), args).ok()?; + let callee = + Instance::try_resolve(tcx, inliner.typing_env(), def_id, args).ok().flatten()?; + + if let InstanceKind::Virtual(..) | InstanceKind::Intrinsic(_) = callee.def { + return None; + } - false + if inliner.history().contains(&callee.def_id()) { + return None; + } + + let fn_sig = tcx.fn_sig(def_id).instantiate(tcx, args); + + // Additionally, check that the body that we're inlining actually agrees + // with the ABI of the trait that the item comes from. + if let InstanceKind::Item(instance_def_id) = callee.def + && tcx.def_kind(instance_def_id) == DefKind::AssocFn + && let instance_fn_sig = tcx.fn_sig(instance_def_id).skip_binder() + && instance_fn_sig.abi() != fn_sig.abi() + { + return None; + } + + let source_info = SourceInfo { span: fn_span, ..terminator.source_info }; + + return Some(CallSite { callee, fn_sig, block: bb, source_info }); } + } - let dest = if dest_needs_borrow(destination) { - trace!("creating temp for return destination"); - let dest = Rvalue::Ref( - self.tcx.lifetimes.re_erased, - BorrowKind::Mut { kind: MutBorrowKind::Default }, - destination, - ); - let dest_ty = dest.ty(caller_body, self.tcx); - let temp = - Place::from(self.new_call_temp(caller_body, callsite, dest_ty, return_block)); - caller_body[callsite.block].statements.push(Statement { - source_info: callsite.source_info, - kind: StatementKind::Assign(Box::new((temp, dest))), - }); - self.tcx.mk_place_deref(temp) - } else { - destination - }; + None +} - // Always create a local to hold the destination, as `RETURN_PLACE` may appear - // where a full `Place` is not allowed. - let (remap_destination, destination_local) = if let Some(d) = dest.as_local() { - (false, d) - } else { - ( - true, - self.new_call_temp( - caller_body, - callsite, - destination.ty(caller_body, self.tcx).ty, - return_block, - ), - ) - }; +/// Attempts to inline a callsite into the caller body. When successful returns basic blocks +/// containing the inlined body. Otherwise returns an error describing why inlining didn't take +/// place. +fn try_inlining<'tcx, I: Inliner<'tcx>>( + inliner: &I, + caller_body: &mut Body<'tcx>, + callsite: &CallSite<'tcx>, +) -> Result, &'static str> { + let tcx = inliner.tcx(); + check_mir_is_available(inliner, caller_body, callsite.callee)?; + + let callee_attrs = tcx.codegen_fn_attrs(callsite.callee.def_id()); + let cross_crate_inlinable = tcx.cross_crate_inlinable(callsite.callee.def_id()); + check_codegen_attributes(inliner, callsite, callee_attrs, cross_crate_inlinable)?; + + // Intrinsic fallback bodies are automatically made cross-crate inlineable, + // but at this stage we don't know whether codegen knows the intrinsic, + // so just conservatively don't inline it. + if tcx.has_attr(callsite.callee.def_id(), sym::rustc_intrinsic) { + return Err("callee is an intrinsic"); + } - // Copy the arguments if needed. - let args = self.make_call_args(args, callsite, caller_body, &callee_body, return_block); + let terminator = caller_body[callsite.block].terminator.as_ref().unwrap(); + let TerminatorKind::Call { args, destination, .. } = &terminator.kind else { bug!() }; + let destination_ty = destination.ty(&caller_body.local_decls, tcx).ty; + for arg in args { + if !arg.node.ty(&caller_body.local_decls, tcx).is_sized(tcx, inliner.typing_env()) { + // We do not allow inlining functions with unsized params. Inlining these functions + // could create unsized locals, which are unsound and being phased out. + return Err("call has unsized argument"); + } + } - let mut integrator = Integrator { - args: &args, - new_locals: Local::new(caller_body.local_decls.len()).., - new_scopes: SourceScope::new(caller_body.source_scopes.len()).., - new_blocks: BasicBlock::new(caller_body.basic_blocks.len()).., - destination: destination_local, - callsite_scope: caller_body.source_scopes[callsite.source_info.scope].clone(), - callsite, - cleanup_block: unwind, - in_cleanup_block: false, - return_block, - tcx: self.tcx, - always_live_locals: BitSet::new_filled(callee_body.local_decls.len()), + let callee_body = try_instance_mir(tcx, callsite.callee.def)?; + inliner.check_callee_mir_body(callsite, callee_body, callee_attrs, cross_crate_inlinable)?; + + let Ok(callee_body) = callsite.callee.try_instantiate_mir_and_normalize_erasing_regions( + tcx, + inliner.typing_env(), + ty::EarlyBinder::bind(callee_body.clone()), + ) else { + debug!("failed to normalize callee body"); + return Err("implementation limitation"); + }; + + // Normally, this shouldn't be required, but trait normalization failure can create a + // validation ICE. + if !validate_types(tcx, inliner.typing_env(), &callee_body, &caller_body).is_empty() { + debug!("failed to validate callee body"); + return Err("implementation limitation"); + } + + // Check call signature compatibility. + // Normally, this shouldn't be required, but trait normalization failure can create a + // validation ICE. + let output_type = callee_body.return_ty(); + if !util::sub_types(tcx, inliner.typing_env(), output_type, destination_ty) { + trace!(?output_type, ?destination_ty); + debug!("failed to normalize return type"); + return Err("implementation limitation"); + } + if callsite.fn_sig.abi() == ExternAbi::RustCall { + // FIXME: Don't inline user-written `extern "rust-call"` functions, + // since this is generally perf-negative on rustc, and we hope that + // LLVM will inline these functions instead. + if callee_body.spread_arg.is_some() { + return Err("user-written rust-call functions"); + } + + let (self_arg, arg_tuple) = match &args[..] { + [arg_tuple] => (None, arg_tuple), + [self_arg, arg_tuple] => (Some(self_arg), arg_tuple), + _ => bug!("Expected `rust-call` to have 1 or 2 args"), }; - // Map all `Local`s, `SourceScope`s and `BasicBlock`s to new ones - // (or existing ones, in a few special cases) in the caller. - integrator.visit_body(&mut callee_body); + let self_arg_ty = self_arg.map(|self_arg| self_arg.node.ty(&caller_body.local_decls, tcx)); - // If there are any locals without storage markers, give them storage only for the - // duration of the call. - for local in callee_body.vars_and_temps_iter() { - if integrator.always_live_locals.contains(local) { - let new_local = integrator.map_local(local); - caller_body[callsite.block].statements.push(Statement { - source_info: callsite.source_info, - kind: StatementKind::StorageLive(new_local), - }); + let arg_tuple_ty = arg_tuple.node.ty(&caller_body.local_decls, tcx); + let ty::Tuple(arg_tuple_tys) = *arg_tuple_ty.kind() else { + bug!("Closure arguments are not passed as a tuple"); + }; + + for (arg_ty, input) in + self_arg_ty.into_iter().chain(arg_tuple_tys).zip(callee_body.args_iter()) + { + let input_type = callee_body.local_decls[input].ty; + if !util::sub_types(tcx, inliner.typing_env(), input_type, arg_ty) { + trace!(?arg_ty, ?input_type); + debug!("failed to normalize tuple argument type"); + return Err("implementation limitation"); } } - if let Some(block) = return_block { - // To avoid repeated O(n) insert, push any new statements to the end and rotate - // the slice once. - let mut n = 0; - if remap_destination { - caller_body[block].statements.push(Statement { - source_info: callsite.source_info, - kind: StatementKind::Assign(Box::new(( - dest, - Rvalue::Use(Operand::Move(destination_local.into())), - ))), - }); - n += 1; + } else { + for (arg, input) in args.iter().zip(callee_body.args_iter()) { + let input_type = callee_body.local_decls[input].ty; + let arg_ty = arg.node.ty(&caller_body.local_decls, tcx); + if !util::sub_types(tcx, inliner.typing_env(), input_type, arg_ty) { + trace!(?arg_ty, ?input_type); + debug!("failed to normalize argument type"); + return Err("implementation limitation"); } - for local in callee_body.vars_and_temps_iter().rev() { - if integrator.always_live_locals.contains(local) { - let new_local = integrator.map_local(local); - caller_body[block].statements.push(Statement { - source_info: callsite.source_info, - kind: StatementKind::StorageDead(new_local), - }); - n += 1; - } + } + } + + let old_blocks = caller_body.basic_blocks.next_index(); + inline_call(inliner, caller_body, callsite, callee_body); + let new_blocks = old_blocks..caller_body.basic_blocks.next_index(); + + Ok(new_blocks) +} + +fn check_mir_is_available<'tcx, I: Inliner<'tcx>>( + inliner: &I, + caller_body: &Body<'tcx>, + callee: Instance<'tcx>, +) -> Result<(), &'static str> { + let caller_def_id = caller_body.source.def_id(); + let callee_def_id = callee.def_id(); + if callee_def_id == caller_def_id { + return Err("self-recursion"); + } + + match callee.def { + InstanceKind::Item(_) => { + // If there is no MIR available (either because it was not in metadata or + // because it has no MIR because it's an extern function), then the inliner + // won't cause cycles on this. + if !inliner.tcx().is_mir_available(callee_def_id) { + debug!("item MIR unavailable"); + return Err("implementation limitation"); } - caller_body[block].statements.rotate_right(n); + } + // These have no own callable MIR. + InstanceKind::Intrinsic(_) | InstanceKind::Virtual(..) => { + debug!("instance without MIR (intrinsic / virtual)"); + return Err("implementation limitation"); } - // Insert all of the (mapped) parts of the callee body into the caller. - caller_body.local_decls.extend(callee_body.drain_vars_and_temps()); - caller_body.source_scopes.append(&mut callee_body.source_scopes); - if self - .tcx - .sess - .opts - .unstable_opts - .inline_mir_preserve_debug - .unwrap_or(self.tcx.sess.opts.debuginfo != DebugInfo::None) - { - // Note that we need to preserve these in the standard library so that - // people working on rust can build with or without debuginfo while - // still getting consistent results from the mir-opt tests. - caller_body.var_debug_info.append(&mut callee_body.var_debug_info); + // FIXME(#127030): `ConstParamHasTy` has bad interactions with + // the drop shim builder, which does not evaluate predicates in + // the correct param-env for types being dropped. Stall resolving + // the MIR for this instance until all of its const params are + // substituted. + InstanceKind::DropGlue(_, Some(ty)) if ty.has_type_flags(TypeFlags::HAS_CT_PARAM) => { + debug!("still needs substitution"); + return Err("implementation limitation"); } - caller_body.basic_blocks_mut().append(callee_body.basic_blocks_mut()); - caller_body[callsite.block].terminator = Some(Terminator { - source_info: callsite.source_info, - kind: TerminatorKind::Goto { target: integrator.map_block(START_BLOCK) }, - }); + // This cannot result in an immediate cycle since the callee MIR is a shim, which does + // not get any optimizations run on it. Any subsequent inlining may cause cycles, but we + // do not need to catch this here, we can wait until the inliner decides to continue + // inlining a second time. + InstanceKind::VTableShim(_) + | InstanceKind::ReifyShim(..) + | InstanceKind::FnPtrShim(..) + | InstanceKind::ClosureOnceShim { .. } + | InstanceKind::ConstructCoroutineInClosureShim { .. } + | InstanceKind::DropGlue(..) + | InstanceKind::CloneShim(..) + | InstanceKind::ThreadLocalShim(..) + | InstanceKind::FnPtrAddrShim(..) + | InstanceKind::AsyncDropGlueCtorShim(..) => return Ok(()), + } - // Copy required constants from the callee_body into the caller_body. Although we are only - // pushing unevaluated consts to `required_consts`, here they may have been evaluated - // because we are calling `instantiate_and_normalize_erasing_regions` -- so we filter again. - caller_body.required_consts.as_mut().unwrap().extend( - callee_body.required_consts().into_iter().filter(|ct| ct.const_.is_required_const()), - ); - // Now that we incorporated the callee's `required_consts`, we can remove the callee from - // `mentioned_items` -- but we have to take their `mentioned_items` in return. This does - // some extra work here to save the monomorphization collector work later. It helps a lot, - // since monomorphization can avoid a lot of work when the "mentioned items" are similar to - // the actually used items. By doing this we can entirely avoid visiting the callee! - // We need to reconstruct the `required_item` for the callee so that we can find and - // remove it. - let callee_item = MentionedItem::Fn(func.ty(caller_body, self.tcx)); - let caller_mentioned_items = caller_body.mentioned_items.as_mut().unwrap(); - if let Some(idx) = caller_mentioned_items.iter().position(|item| item.node == callee_item) { - // We found the callee, so remove it and add its items instead. - caller_mentioned_items.remove(idx); - caller_mentioned_items.extend(callee_body.mentioned_items()); - } else { - // If we can't find the callee, there's no point in adding its items. Probably it - // already got removed by being inlined elsewhere in the same function, so we already - // took its items. + if inliner.tcx().is_constructor(callee_def_id) { + trace!("constructors always have MIR"); + // Constructor functions cannot cause a query cycle. + return Ok(()); + } + + if callee_def_id.is_local() { + // If we know for sure that the function we're calling will itself try to + // call us, then we avoid inlining that function. + if inliner.tcx().mir_callgraph_reachable((callee, caller_def_id.expect_local())) { + debug!("query cycle avoidance"); + return Err("caller might be reachable from callee"); } + + Ok(()) + } else { + // This cannot result in an immediate cycle since the callee MIR is from another crate + // and is already optimized. Any subsequent inlining may cause cycles, but we do + // not need to catch this here, we can wait until the inliner decides to continue + // inlining a second time. + trace!("functions from other crates always have MIR"); + Ok(()) } +} - fn make_call_args( - &self, - args: Box<[Spanned>]>, - callsite: &CallSite<'tcx>, - caller_body: &mut Body<'tcx>, - callee_body: &Body<'tcx>, - return_block: Option, - ) -> Box<[Local]> { - let tcx = self.tcx; - - // There is a bit of a mismatch between the *caller* of a closure and the *callee*. - // The caller provides the arguments wrapped up in a tuple: - // - // tuple_tmp = (a, b, c) - // Fn::call(closure_ref, tuple_tmp) - // - // meanwhile the closure body expects the arguments (here, `a`, `b`, and `c`) - // as distinct arguments. (This is the "rust-call" ABI hack.) Normally, codegen has - // the job of unpacking this tuple. But here, we are codegen. =) So we want to create - // a vector like - // - // [closure_ref, tuple_tmp.0, tuple_tmp.1, tuple_tmp.2] - // - // Except for one tiny wrinkle: we don't actually want `tuple_tmp.0`. It's more convenient - // if we "spill" that into *another* temporary, so that we can map the argument - // variable in the callee MIR directly to an argument variable on our side. - // So we introduce temporaries like: - // - // tmp0 = tuple_tmp.0 - // tmp1 = tuple_tmp.1 - // tmp2 = tuple_tmp.2 - // - // and the vector is `[closure_ref, tmp0, tmp1, tmp2]`. - if callsite.fn_sig.abi() == ExternAbi::RustCall && callee_body.spread_arg.is_none() { - // FIXME(edition_2024): switch back to a normal method call. - let mut args = <_>::into_iter(args); - let self_ = self.create_temp_if_necessary( - args.next().unwrap().node, - callsite, - caller_body, - return_block, - ); - let tuple = self.create_temp_if_necessary( - args.next().unwrap().node, - callsite, - caller_body, - return_block, - ); - assert!(args.next().is_none()); - - let tuple = Place::from(tuple); - let ty::Tuple(tuple_tys) = tuple.ty(caller_body, tcx).ty.kind() else { - bug!("Closure arguments are not passed as a tuple"); - }; +/// Returns an error if inlining is not possible based on codegen attributes alone. A success +/// indicates that inlining decision should be based on other criteria. +fn check_codegen_attributes<'tcx, I: Inliner<'tcx>>( + inliner: &I, + callsite: &CallSite<'tcx>, + callee_attrs: &CodegenFnAttrs, + cross_crate_inlinable: bool, +) -> Result<(), &'static str> { + let tcx = inliner.tcx(); + if tcx.has_attr(callsite.callee.def_id(), sym::rustc_no_mir_inline) { + return Err("#[rustc_no_mir_inline]"); + } - // The `closure_ref` in our example above. - let closure_ref_arg = iter::once(self_); + if let InlineAttr::Never = callee_attrs.inline { + return Err("never inline attribute"); + } - // The `tmp0`, `tmp1`, and `tmp2` in our example above. - let tuple_tmp_args = tuple_tys.iter().enumerate().map(|(i, ty)| { - // This is e.g., `tuple_tmp.0` in our example above. - let tuple_field = Operand::Move(tcx.mk_place_field(tuple, FieldIdx::new(i), ty)); + // FIXME(#127234): Coverage instrumentation currently doesn't handle inlined + // MIR correctly when Modified Condition/Decision Coverage is enabled. + if tcx.sess.instrument_coverage_mcdc() { + return Err("incompatible with MC/DC coverage"); + } - // Spill to a local to make e.g., `tmp0`. - self.create_temp_if_necessary(tuple_field, callsite, caller_body, return_block) - }); + // Reachability pass defines which functions are eligible for inlining. Generally inlining + // other functions is incorrect because they could reference symbols that aren't exported. + let is_generic = callsite.callee.args.non_erasable_generics().next().is_some(); + if !is_generic && !cross_crate_inlinable { + return Err("not exported"); + } - closure_ref_arg.chain(tuple_tmp_args).collect() - } else { - // FIXME(edition_2024): switch back to a normal method call. - <_>::into_iter(args) - .map(|a| self.create_temp_if_necessary(a.node, callsite, caller_body, return_block)) - .collect() - } + if callsite.fn_sig.c_variadic() { + return Err("C variadic"); } - /// If `arg` is already a temporary, returns it. Otherwise, introduces a fresh - /// temporary `T` and an instruction `T = arg`, and returns `T`. - fn create_temp_if_necessary( - &self, - arg: Operand<'tcx>, - callsite: &CallSite<'tcx>, - caller_body: &mut Body<'tcx>, - return_block: Option, - ) -> Local { - // Reuse the operand if it is a moved temporary. - if let Operand::Move(place) = &arg - && let Some(local) = place.as_local() - && caller_body.local_kind(local) == LocalKind::Temp - { - return local; - } + if callee_attrs.flags.contains(CodegenFnAttrFlags::COLD) { + return Err("cold"); + } - // Otherwise, create a temporary for the argument. - trace!("creating temp for argument {:?}", arg); - let arg_ty = arg.ty(caller_body, self.tcx); - let local = self.new_call_temp(caller_body, callsite, arg_ty, return_block); - caller_body[callsite.block].statements.push(Statement { - source_info: callsite.source_info, - kind: StatementKind::Assign(Box::new((Place::from(local), Rvalue::Use(arg)))), - }); - local + let codegen_fn_attrs = tcx.codegen_fn_attrs(inliner.caller_def_id()); + if callee_attrs.no_sanitize != codegen_fn_attrs.no_sanitize { + return Err("incompatible sanitizer set"); } - /// Introduces a new temporary into the caller body that is live for the duration of the call. - fn new_call_temp( - &self, - caller_body: &mut Body<'tcx>, - callsite: &CallSite<'tcx>, - ty: Ty<'tcx>, - return_block: Option, - ) -> Local { - let local = caller_body.local_decls.push(LocalDecl::new(ty, callsite.source_info.span)); + // Two functions are compatible if the callee has no attribute (meaning + // that it's codegen agnostic), or sets an attribute that is identical + // to this function's attribute. + if callee_attrs.instruction_set.is_some() + && callee_attrs.instruction_set != codegen_fn_attrs.instruction_set + { + return Err("incompatible instruction set"); + } + + let callee_feature_names = callee_attrs.target_features.iter().map(|f| f.name); + let this_feature_names = codegen_fn_attrs.target_features.iter().map(|f| f.name); + if callee_feature_names.ne(this_feature_names) { + // In general it is not correct to inline a callee with target features that are a + // subset of the caller. This is because the callee might contain calls, and the ABI of + // those calls depends on the target features of the surrounding function. By moving a + // `Call` terminator from one MIR body to another with more target features, we might + // change the ABI of that call! + return Err("incompatible target features"); + } + + Ok(()) +} +fn inline_call<'tcx, I: Inliner<'tcx>>( + inliner: &I, + caller_body: &mut Body<'tcx>, + callsite: &CallSite<'tcx>, + mut callee_body: Body<'tcx>, +) { + let tcx = inliner.tcx(); + let terminator = caller_body[callsite.block].terminator.take().unwrap(); + let TerminatorKind::Call { func, args, destination, unwind, target, .. } = terminator.kind + else { + bug!("unexpected terminator kind {:?}", terminator.kind); + }; + + let return_block = if let Some(block) = target { + // Prepare a new block for code that should execute when call returns. We don't use + // target block directly since it might have other predecessors. + let mut data = BasicBlockData::new(Some(Terminator { + source_info: terminator.source_info, + kind: TerminatorKind::Goto { target: block }, + })); + data.is_cleanup = caller_body[block].is_cleanup; + Some(caller_body.basic_blocks_mut().push(data)) + } else { + None + }; + + // If the call is something like `a[*i] = f(i)`, where + // `i : &mut usize`, then just duplicating the `a[*i]` + // Place could result in two different locations if `f` + // writes to `i`. To prevent this we need to create a temporary + // borrow of the place and pass the destination as `*temp` instead. + fn dest_needs_borrow(place: Place<'_>) -> bool { + for elem in place.projection.iter() { + match elem { + ProjectionElem::Deref | ProjectionElem::Index(_) => return true, + _ => {} + } + } + + false + } + + let dest = if dest_needs_borrow(destination) { + trace!("creating temp for return destination"); + let dest = Rvalue::Ref( + tcx.lifetimes.re_erased, + BorrowKind::Mut { kind: MutBorrowKind::Default }, + destination, + ); + let dest_ty = dest.ty(caller_body, tcx); + let temp = Place::from(new_call_temp(caller_body, callsite, dest_ty, return_block)); caller_body[callsite.block].statements.push(Statement { source_info: callsite.source_info, - kind: StatementKind::StorageLive(local), + kind: StatementKind::Assign(Box::new((temp, dest))), }); + tcx.mk_place_deref(temp) + } else { + destination + }; + + // Always create a local to hold the destination, as `RETURN_PLACE` may appear + // where a full `Place` is not allowed. + let (remap_destination, destination_local) = if let Some(d) = dest.as_local() { + (false, d) + } else { + ( + true, + new_call_temp(caller_body, callsite, destination.ty(caller_body, tcx).ty, return_block), + ) + }; - if let Some(block) = return_block { - caller_body[block].statements.insert(0, Statement { + // Copy the arguments if needed. + let args = make_call_args(inliner, args, callsite, caller_body, &callee_body, return_block); + + let mut integrator = Integrator { + args: &args, + new_locals: Local::new(caller_body.local_decls.len()).., + new_scopes: SourceScope::new(caller_body.source_scopes.len()).., + new_blocks: BasicBlock::new(caller_body.basic_blocks.len()).., + destination: destination_local, + callsite_scope: caller_body.source_scopes[callsite.source_info.scope].clone(), + callsite, + cleanup_block: unwind, + in_cleanup_block: false, + return_block, + tcx, + always_live_locals: BitSet::new_filled(callee_body.local_decls.len()), + }; + + // Map all `Local`s, `SourceScope`s and `BasicBlock`s to new ones + // (or existing ones, in a few special cases) in the caller. + integrator.visit_body(&mut callee_body); + + // If there are any locals without storage markers, give them storage only for the + // duration of the call. + for local in callee_body.vars_and_temps_iter() { + if integrator.always_live_locals.contains(local) { + let new_local = integrator.map_local(local); + caller_body[callsite.block].statements.push(Statement { source_info: callsite.source_info, - kind: StatementKind::StorageDead(local), + kind: StatementKind::StorageLive(new_local), }); } + } + if let Some(block) = return_block { + // To avoid repeated O(n) insert, push any new statements to the end and rotate + // the slice once. + let mut n = 0; + if remap_destination { + caller_body[block].statements.push(Statement { + source_info: callsite.source_info, + kind: StatementKind::Assign(Box::new(( + dest, + Rvalue::Use(Operand::Move(destination_local.into())), + ))), + }); + n += 1; + } + for local in callee_body.vars_and_temps_iter().rev() { + if integrator.always_live_locals.contains(local) { + let new_local = integrator.map_local(local); + caller_body[block].statements.push(Statement { + source_info: callsite.source_info, + kind: StatementKind::StorageDead(new_local), + }); + n += 1; + } + } + caller_body[block].statements.rotate_right(n); + } + + // Insert all of the (mapped) parts of the callee body into the caller. + caller_body.local_decls.extend(callee_body.drain_vars_and_temps()); + caller_body.source_scopes.append(&mut callee_body.source_scopes); + if tcx + .sess + .opts + .unstable_opts + .inline_mir_preserve_debug + .unwrap_or(tcx.sess.opts.debuginfo != DebugInfo::None) + { + // Note that we need to preserve these in the standard library so that + // people working on rust can build with or without debuginfo while + // still getting consistent results from the mir-opt tests. + caller_body.var_debug_info.append(&mut callee_body.var_debug_info); + } + caller_body.basic_blocks_mut().append(callee_body.basic_blocks_mut()); + + caller_body[callsite.block].terminator = Some(Terminator { + source_info: callsite.source_info, + kind: TerminatorKind::Goto { target: integrator.map_block(START_BLOCK) }, + }); + + // Copy required constants from the callee_body into the caller_body. Although we are only + // pushing unevaluated consts to `required_consts`, here they may have been evaluated + // because we are calling `instantiate_and_normalize_erasing_regions` -- so we filter again. + caller_body.required_consts.as_mut().unwrap().extend( + callee_body.required_consts().into_iter().filter(|ct| ct.const_.is_required_const()), + ); + // Now that we incorporated the callee's `required_consts`, we can remove the callee from + // `mentioned_items` -- but we have to take their `mentioned_items` in return. This does + // some extra work here to save the monomorphization collector work later. It helps a lot, + // since monomorphization can avoid a lot of work when the "mentioned items" are similar to + // the actually used items. By doing this we can entirely avoid visiting the callee! + // We need to reconstruct the `required_item` for the callee so that we can find and + // remove it. + let callee_item = MentionedItem::Fn(func.ty(caller_body, tcx)); + let caller_mentioned_items = caller_body.mentioned_items.as_mut().unwrap(); + if let Some(idx) = caller_mentioned_items.iter().position(|item| item.node == callee_item) { + // We found the callee, so remove it and add its items instead. + caller_mentioned_items.remove(idx); + caller_mentioned_items.extend(callee_body.mentioned_items()); + } else { + // If we can't find the callee, there's no point in adding its items. Probably it + // already got removed by being inlined elsewhere in the same function, so we already + // took its items. + } +} + +fn make_call_args<'tcx, I: Inliner<'tcx>>( + inliner: &I, + args: Box<[Spanned>]>, + callsite: &CallSite<'tcx>, + caller_body: &mut Body<'tcx>, + callee_body: &Body<'tcx>, + return_block: Option, +) -> Box<[Local]> { + let tcx = inliner.tcx(); + + // There is a bit of a mismatch between the *caller* of a closure and the *callee*. + // The caller provides the arguments wrapped up in a tuple: + // + // tuple_tmp = (a, b, c) + // Fn::call(closure_ref, tuple_tmp) + // + // meanwhile the closure body expects the arguments (here, `a`, `b`, and `c`) + // as distinct arguments. (This is the "rust-call" ABI hack.) Normally, codegen has + // the job of unpacking this tuple. But here, we are codegen. =) So we want to create + // a vector like + // + // [closure_ref, tuple_tmp.0, tuple_tmp.1, tuple_tmp.2] + // + // Except for one tiny wrinkle: we don't actually want `tuple_tmp.0`. It's more convenient + // if we "spill" that into *another* temporary, so that we can map the argument + // variable in the callee MIR directly to an argument variable on our side. + // So we introduce temporaries like: + // + // tmp0 = tuple_tmp.0 + // tmp1 = tuple_tmp.1 + // tmp2 = tuple_tmp.2 + // + // and the vector is `[closure_ref, tmp0, tmp1, tmp2]`. + if callsite.fn_sig.abi() == ExternAbi::RustCall && callee_body.spread_arg.is_none() { + // FIXME(edition_2024): switch back to a normal method call. + let mut args = <_>::into_iter(args); + let self_ = create_temp_if_necessary( + inliner, + args.next().unwrap().node, + callsite, + caller_body, + return_block, + ); + let tuple = create_temp_if_necessary( + inliner, + args.next().unwrap().node, + callsite, + caller_body, + return_block, + ); + assert!(args.next().is_none()); + + let tuple = Place::from(tuple); + let ty::Tuple(tuple_tys) = tuple.ty(caller_body, tcx).ty.kind() else { + bug!("Closure arguments are not passed as a tuple"); + }; + + // The `closure_ref` in our example above. + let closure_ref_arg = iter::once(self_); + + // The `tmp0`, `tmp1`, and `tmp2` in our example above. + let tuple_tmp_args = tuple_tys.iter().enumerate().map(|(i, ty)| { + // This is e.g., `tuple_tmp.0` in our example above. + let tuple_field = Operand::Move(tcx.mk_place_field(tuple, FieldIdx::new(i), ty)); + + // Spill to a local to make e.g., `tmp0`. + create_temp_if_necessary(inliner, tuple_field, callsite, caller_body, return_block) + }); + + closure_ref_arg.chain(tuple_tmp_args).collect() + } else { + // FIXME(edition_2024): switch back to a normal method call. + <_>::into_iter(args) + .map(|a| create_temp_if_necessary(inliner, a.node, callsite, caller_body, return_block)) + .collect() + } +} - local +/// If `arg` is already a temporary, returns it. Otherwise, introduces a fresh temporary `T` and an +/// instruction `T = arg`, and returns `T`. +fn create_temp_if_necessary<'tcx, I: Inliner<'tcx>>( + inliner: &I, + arg: Operand<'tcx>, + callsite: &CallSite<'tcx>, + caller_body: &mut Body<'tcx>, + return_block: Option, +) -> Local { + // Reuse the operand if it is a moved temporary. + if let Operand::Move(place) = &arg + && let Some(local) = place.as_local() + && caller_body.local_kind(local) == LocalKind::Temp + { + return local; + } + + // Otherwise, create a temporary for the argument. + trace!("creating temp for argument {:?}", arg); + let arg_ty = arg.ty(caller_body, inliner.tcx()); + let local = new_call_temp(caller_body, callsite, arg_ty, return_block); + caller_body[callsite.block].statements.push(Statement { + source_info: callsite.source_info, + kind: StatementKind::Assign(Box::new((Place::from(local), Rvalue::Use(arg)))), + }); + local +} + +/// Introduces a new temporary into the caller body that is live for the duration of the call. +fn new_call_temp<'tcx>( + caller_body: &mut Body<'tcx>, + callsite: &CallSite<'tcx>, + ty: Ty<'tcx>, + return_block: Option, +) -> Local { + let local = caller_body.local_decls.push(LocalDecl::new(ty, callsite.source_info.span)); + + caller_body[callsite.block].statements.push(Statement { + source_info: callsite.source_info, + kind: StatementKind::StorageLive(local), + }); + + if let Some(block) = return_block { + caller_body[block].statements.insert(0, Statement { + source_info: callsite.source_info, + kind: StatementKind::StorageDead(local), + }); } + + local } /** diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs index f0fcb44603b15..6b2b7ee5241fe 100644 --- a/compiler/rustc_mir_transform/src/lib.rs +++ b/compiler/rustc_mir_transform/src/lib.rs @@ -141,7 +141,7 @@ declare_passes! { mod gvn : GVN; // Made public so that `mir_drops_elaborated_and_const_checked` can be overridden // by custom rustc drivers, running all the steps by themselves. See #114628. - pub mod inline : Inline; + pub mod inline : Inline, ForceInline; mod instsimplify : InstSimplify { BeforeInline, AfterSimplifyCfg }; mod jump_threading : JumpThreading; mod known_panics_lint : KnownPanicsLint; @@ -485,7 +485,9 @@ fn mir_drops_elaborated_and_const_checked(tcx: TyCtxt<'_>, def: LocalDefId) -> & let is_fn_like = tcx.def_kind(def).is_fn_like(); if is_fn_like { // Do not compute the mir call graph without said call graph actually being used. - if pm::should_run_pass(tcx, &inline::Inline) { + if pm::should_run_pass(tcx, &inline::Inline) + || inline::ForceInline::should_run_pass_for_callee(tcx, def.to_def_id()) + { tcx.ensure_with_value().mir_inliner_callees(ty::InstanceKind::Item(def.to_def_id())); } } @@ -662,6 +664,8 @@ fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { // Perform instsimplify before inline to eliminate some trivial calls (like clone // shims). &instsimplify::InstSimplify::BeforeInline, + // Perform inlining of `#[rustc_force_inline]`-annotated callees. + &inline::ForceInline, // Perform inlining, which may add a lot of code. &inline::Inline, // Code from other crates may have storage markers, so this needs to happen after diff --git a/compiler/rustc_mir_transform/src/pass_manager.rs b/compiler/rustc_mir_transform/src/pass_manager.rs index 8a45ce0762d59..c3f0a989ce105 100644 --- a/compiler/rustc_mir_transform/src/pass_manager.rs +++ b/compiler/rustc_mir_transform/src/pass_manager.rs @@ -79,6 +79,12 @@ pub(super) trait MirPass<'tcx> { true } + /// Returns `true` if this pass can be overridden by `-Zenable-mir-passes`. This should be + /// true for basically every pass other than those that are necessary for correctness. + fn can_be_overridden(&self) -> bool { + true + } + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>); fn is_mir_dump_enabled(&self) -> bool { @@ -176,6 +182,10 @@ where { let name = pass.name(); + if !pass.can_be_overridden() { + return pass.is_enabled(tcx.sess); + } + let overridden_passes = &tcx.sess.opts.unstable_opts.mir_enable_passes; let overridden = overridden_passes.iter().rev().find(|(s, _)| s == &*name).map(|(_name, polarity)| { diff --git a/compiler/rustc_mir_transform/src/validate.rs b/compiler/rustc_mir_transform/src/validate.rs index 404fbb6b83937..927cbe998527e 100644 --- a/compiler/rustc_mir_transform/src/validate.rs +++ b/compiler/rustc_mir_transform/src/validate.rs @@ -1,6 +1,7 @@ //! Validates the MIR to ensure that invariants are upheld. use rustc_abi::{ExternAbi, FIRST_VARIANT, Size}; +use rustc_attr::InlineAttr; use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_hir::LangItem; use rustc_index::IndexVec; @@ -79,7 +80,7 @@ impl<'tcx> crate::MirPass<'tcx> for Validator { cfg_checker.fail(location, msg); } - if let MirPhase::Runtime(_) = body.phase { + if let MirPhase::Runtime(phase) = body.phase { if let ty::InstanceKind::Item(_) = body.source.instance { if body.has_free_regions() { cfg_checker.fail( @@ -88,6 +89,27 @@ impl<'tcx> crate::MirPass<'tcx> for Validator { ); } } + + if phase >= RuntimePhase::Optimized + && body + .basic_blocks + .iter() + .filter_map(|bb| match &bb.terminator().kind { + TerminatorKind::Call { func, .. } + | TerminatorKind::TailCall { func, .. } => Some(func), + _ => None, + }) + .filter_map(|func| match func.ty(&body.local_decls, tcx).kind() { + ty::FnDef(did, ..) => Some(did), + _ => None, + }) + .any(|did| matches!(tcx.codegen_fn_attrs(did).inline, InlineAttr::Force { .. })) + { + cfg_checker.fail( + Location::START, + "`#[rustc_force_inline]`-annotated function not inlined", + ); + } } } } diff --git a/compiler/rustc_monomorphize/src/collector.rs b/compiler/rustc_monomorphize/src/collector.rs index 480d82c1a385b..cb0493df77fe0 100644 --- a/compiler/rustc_monomorphize/src/collector.rs +++ b/compiler/rustc_monomorphize/src/collector.rs @@ -960,6 +960,14 @@ fn should_codegen_locally<'tcx>(tcx: TyCtxtAt<'tcx>, instance: Instance<'tcx>) - return false; } + if tcx.def_kind(def_id).has_codegen_attrs() + && matches!(tcx.codegen_fn_attrs(def_id).inline, rustc_attr::InlineAttr::Force { .. }) + { + // `#[rustc_force_inline]` items should never be codegened. This should be caught by + // the MIR validator. + return false; + } + if def_id.is_local() { // Local items cannot be referred to locally without monomorphizing them locally. return true; diff --git a/compiler/rustc_passes/messages.ftl b/compiler/rustc_passes/messages.ftl index 7a0a518bb513a..dad69df41ea46 100644 --- a/compiler/rustc_passes/messages.ftl +++ b/compiler/rustc_passes/messages.ftl @@ -646,6 +646,14 @@ passes_rustc_allow_const_fn_unstable = passes_rustc_dirty_clean = attribute requires -Z query-dep-graph to be enabled +passes_rustc_force_inline = + attribute should be applied to a function + .label = not a function definition + +passes_rustc_force_inline_coro = + attribute cannot be applied to a `async`, `gen` or `async gen` function + .label = `async`, `gen` or `async gen` function + passes_rustc_layout_scalar_valid_range_arg = expected exactly one integer literal argument diff --git a/compiler/rustc_passes/src/check_attr.rs b/compiler/rustc_passes/src/check_attr.rs index 074fe77324faf..ae2d51f0aa1a7 100644 --- a/compiler/rustc_passes/src/check_attr.rs +++ b/compiler/rustc_passes/src/check_attr.rs @@ -249,7 +249,7 @@ impl<'tcx> CheckAttrVisitor<'tcx> { self.check_coroutine(attr, target); } [sym::linkage, ..] => self.check_linkage(attr, span, target), - [sym::rustc_pub_transparent, ..] => self.check_rustc_pub_transparent( attr.span, span, attrs), + [sym::rustc_pub_transparent, ..] => self.check_rustc_pub_transparent(attr.span, span, attrs), [ // ok sym::allow @@ -334,6 +334,7 @@ impl<'tcx> CheckAttrVisitor<'tcx> { self.check_repr(attrs, span, target, item, hir_id); self.check_used(attrs, target, span); + self.check_rustc_force_inline(hir_id, attrs, span, target); } fn inline_attr_str_error_with_macro_def(&self, hir_id: HirId, attr: &Attribute, sym: &str) { @@ -2424,6 +2425,45 @@ impl<'tcx> CheckAttrVisitor<'tcx> { } } + fn check_rustc_force_inline( + &self, + hir_id: HirId, + attrs: &[Attribute], + span: Span, + target: Target, + ) { + let force_inline_attr = attrs.iter().find(|attr| attr.has_name(sym::rustc_force_inline)); + match (target, force_inline_attr) { + (Target::Closure, None) => { + let is_coro = matches!( + self.tcx.hir().expect_expr(hir_id).kind, + hir::ExprKind::Closure(hir::Closure { + kind: hir::ClosureKind::Coroutine(..) + | hir::ClosureKind::CoroutineClosure(..), + .. + }) + ); + let parent_did = self.tcx.hir().get_parent_item(hir_id).to_def_id(); + let parent_span = self.tcx.def_span(parent_did); + let parent_force_inline_attr = + self.tcx.get_attr(parent_did, sym::rustc_force_inline); + if let Some(attr) = parent_force_inline_attr + && is_coro + { + self.dcx().emit_err(errors::RustcForceInlineCoro { + attr_span: attr.span, + span: parent_span, + }); + } + } + (Target::Fn, _) => (), + (_, Some(attr)) => { + self.dcx().emit_err(errors::RustcForceInline { attr_span: attr.span, span }); + } + (_, None) => (), + } + } + /// Checks if `#[autodiff]` is applied to an item other than a function item. fn check_autodiff(&self, _hir_id: HirId, _attr: &Attribute, span: Span, target: Target) { debug!("check_autodiff"); diff --git a/compiler/rustc_passes/src/errors.rs b/compiler/rustc_passes/src/errors.rs index fdc7e1bba2f0e..e63f9ed737a90 100644 --- a/compiler/rustc_passes/src/errors.rs +++ b/compiler/rustc_passes/src/errors.rs @@ -666,6 +666,24 @@ pub(crate) struct RustcPubTransparent { pub span: Span, } +#[derive(Diagnostic)] +#[diag(passes_rustc_force_inline)] +pub(crate) struct RustcForceInline { + #[primary_span] + pub attr_span: Span, + #[label] + pub span: Span, +} + +#[derive(Diagnostic)] +#[diag(passes_rustc_force_inline_coro)] +pub(crate) struct RustcForceInlineCoro { + #[primary_span] + pub attr_span: Span, + #[label] + pub span: Span, +} + #[derive(Diagnostic)] #[diag(passes_link_ordinal)] pub(crate) struct LinkOrdinal { diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index d30b17c9cd8dd..d5acb626fc425 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -1711,6 +1711,7 @@ symbols! { rustc_error, rustc_evaluate_where_clauses, rustc_expected_cgu_reuse, + rustc_force_inline, rustc_has_incoherent_inherent_impls, rustc_hidden_type_of_opaques, rustc_if_this_changed, diff --git a/compiler/rustc_trait_selection/messages.ftl b/compiler/rustc_trait_selection/messages.ftl index 1ab89ecde7a44..b82bb27eb795a 100644 --- a/compiler/rustc_trait_selection/messages.ftl +++ b/compiler/rustc_trait_selection/messages.ftl @@ -251,7 +251,9 @@ trait_selection_no_value_in_rustc_on_unimplemented = this attribute must have a trait_selection_nothing = {""} -trait_selection_oc_cant_coerce = cannot coerce intrinsics to function pointers +trait_selection_oc_cant_coerce_force_inline = + cannot coerce functions which must be inlined to function pointers +trait_selection_oc_cant_coerce_intrinsic = cannot coerce intrinsics to function pointers trait_selection_oc_closure_selfref = closure/coroutine type that references itself trait_selection_oc_const_compat = const not compatible with trait trait_selection_oc_fn_lang_correct_type = {$lang_item_name -> diff --git a/compiler/rustc_trait_selection/src/error_reporting/infer/mod.rs b/compiler/rustc_trait_selection/src/error_reporting/infer/mod.rs index f856a8d7abbe7..dac151c1adced 100644 --- a/compiler/rustc_trait_selection/src/error_reporting/infer/mod.rs +++ b/compiler/rustc_trait_selection/src/error_reporting/infer/mod.rs @@ -2214,7 +2214,7 @@ impl<'tcx> ObligationCause<'tcx> { { FailureCode::Error0644 } - TypeError::IntrinsicCast => FailureCode::Error0308, + TypeError::IntrinsicCast | TypeError::ForceInlineCast => FailureCode::Error0308, _ => FailureCode::Error0308, }, } @@ -2280,8 +2280,11 @@ impl<'tcx> ObligationCause<'tcx> { { ObligationCauseFailureCode::ClosureSelfref { span } } + TypeError::ForceInlineCast => { + ObligationCauseFailureCode::CantCoerceForceInline { span, subdiags } + } TypeError::IntrinsicCast => { - ObligationCauseFailureCode::CantCoerce { span, subdiags } + ObligationCauseFailureCode::CantCoerceIntrinsic { span, subdiags } } _ => ObligationCauseFailureCode::Generic { span, subdiags }, }, diff --git a/compiler/rustc_trait_selection/src/errors.rs b/compiler/rustc_trait_selection/src/errors.rs index 68fe90f0de29a..45c0622a8a66a 100644 --- a/compiler/rustc_trait_selection/src/errors.rs +++ b/compiler/rustc_trait_selection/src/errors.rs @@ -1730,8 +1730,15 @@ pub enum ObligationCauseFailureCode { #[primary_span] span: Span, }, - #[diag(trait_selection_oc_cant_coerce, code = E0308)] - CantCoerce { + #[diag(trait_selection_oc_cant_coerce_force_inline, code = E0308)] + CantCoerceForceInline { + #[primary_span] + span: Span, + #[subdiagnostic] + subdiags: Vec, + }, + #[diag(trait_selection_oc_cant_coerce_intrinsic, code = E0308)] + CantCoerceIntrinsic { #[primary_span] span: Span, #[subdiagnostic] diff --git a/compiler/rustc_type_ir/src/error.rs b/compiler/rustc_type_ir/src/error.rs index 55671b84dbc4f..68b11489ae7c1 100644 --- a/compiler/rustc_type_ir/src/error.rs +++ b/compiler/rustc_type_ir/src/error.rs @@ -51,6 +51,9 @@ pub enum TypeError { ConstMismatch(ExpectedFound), IntrinsicCast, + /// `#[rustc_force_inline]` functions must be inlined and must not be codegened independently, + /// so casting to a function pointer must be prohibited. + ForceInlineCast, /// Safe `#[target_feature]` functions are not assignable to safe function pointers. TargetFeatureCast(I::DefId), } @@ -83,6 +86,7 @@ impl TypeError { | ProjectionMismatched(_) | ExistentialMismatch(_) | ConstMismatch(_) + | ForceInlineCast | IntrinsicCast => true, } } diff --git a/tests/mir-opt/inline/forced.caller.ForceInline.diff b/tests/mir-opt/inline/forced.caller.ForceInline.diff new file mode 100644 index 0000000000000..7b70fd665666a --- /dev/null +++ b/tests/mir-opt/inline/forced.caller.ForceInline.diff @@ -0,0 +1,21 @@ +- // MIR for `caller` before ForceInline ++ // MIR for `caller` after ForceInline + + fn caller() -> () { + let mut _0: (); + let _1: (); ++ scope 1 (inlined callee_forced) { ++ } + + bb0: { + StorageLive(_1); +- _1 = callee_forced() -> [return: bb1, unwind continue]; +- } +- +- bb1: { + StorageDead(_1); + _0 = const (); + return; + } + } + diff --git a/tests/mir-opt/inline/forced.rs b/tests/mir-opt/inline/forced.rs new file mode 100644 index 0000000000000..66b75ff2bb953 --- /dev/null +++ b/tests/mir-opt/inline/forced.rs @@ -0,0 +1,13 @@ +// EMIT_MIT_FOR_EACH_PANIC_STRATEGY +//@ compile-flags: -Copt-level=0 --crate-type=lib +#![feature(rustc_attrs)] + +#[rustc_force_inline] +pub fn callee_forced() {} + +// EMIT_MIR forced.caller.ForceInline.diff +pub fn caller() { + callee_forced(); + // CHECK-LABEL: fn caller( + // CHECK: (inlined callee_forced) +} diff --git a/tests/mir-opt/inline/forced_async.caller.ForceInline.diff b/tests/mir-opt/inline/forced_async.caller.ForceInline.diff new file mode 100644 index 0000000000000..42048bceef84e --- /dev/null +++ b/tests/mir-opt/inline/forced_async.caller.ForceInline.diff @@ -0,0 +1,12 @@ +- // MIR for `caller` before ForceInline ++ // MIR for `caller` after ForceInline + + fn caller() -> {async fn body of caller()} { + let mut _0: {async fn body of caller()}; + + bb0: { + _0 = {coroutine@$DIR/forced_async.rs:10:19: 14:2 (#0)}; + return; + } + } + diff --git a/tests/mir-opt/inline/forced_async.rs b/tests/mir-opt/inline/forced_async.rs new file mode 100644 index 0000000000000..ed49eec01ee3b --- /dev/null +++ b/tests/mir-opt/inline/forced_async.rs @@ -0,0 +1,14 @@ +// EMIT_MIT_FOR_EACH_PANIC_STRATEGY +//@ compile-flags: -Copt-level=0 --crate-type=lib +//@ edition: 2021 +#![feature(rustc_attrs)] + +#[rustc_force_inline] +pub fn callee_forced() {} + +// EMIT_MIR forced_async.caller.ForceInline.diff +async fn caller() { + callee_forced(); + // CHECK-LABEL: fn caller( + // CHECK: (inlined callee_forced) +} diff --git a/tests/mir-opt/inline/forced_closure.caller-{closure#0}.ForceInline.diff b/tests/mir-opt/inline/forced_closure.caller-{closure#0}.ForceInline.diff new file mode 100644 index 0000000000000..8f50c20587d70 --- /dev/null +++ b/tests/mir-opt/inline/forced_closure.caller-{closure#0}.ForceInline.diff @@ -0,0 +1,21 @@ +- // MIR for `caller::{closure#0}` before ForceInline ++ // MIR for `caller::{closure#0}` after ForceInline + + fn caller::{closure#0}(_1: &{closure@$DIR/forced_closure.rs:10:6: 10:8}) -> () { + let mut _0: (); + let _2: (); ++ scope 1 (inlined callee_forced) { ++ } + + bb0: { + StorageLive(_2); +- _2 = callee_forced() -> [return: bb1, unwind continue]; +- } +- +- bb1: { + StorageDead(_2); + _0 = const (); + return; + } + } + diff --git a/tests/mir-opt/inline/forced_closure.rs b/tests/mir-opt/inline/forced_closure.rs new file mode 100644 index 0000000000000..ca5d4a8b7765d --- /dev/null +++ b/tests/mir-opt/inline/forced_closure.rs @@ -0,0 +1,15 @@ +// EMIT_MIT_FOR_EACH_PANIC_STRATEGY +//@ compile-flags: -Copt-level=0 --crate-type=lib +#![feature(rustc_attrs)] + +#[rustc_force_inline] +pub fn callee_forced() {} + +// EMIT_MIR forced_closure.caller-{closure#0}.ForceInline.diff +pub fn caller() { + (|| { + callee_forced(); + // CHECK-LABEL: fn caller::{closure#0}( + // CHECK: (inlined callee_forced) + })(); +} diff --git a/tests/ui/force-inlining/auxiliary/callees.rs b/tests/ui/force-inlining/auxiliary/callees.rs new file mode 100644 index 0000000000000..7468b8f10985d --- /dev/null +++ b/tests/ui/force-inlining/auxiliary/callees.rs @@ -0,0 +1,10 @@ +//@ compile-flags: --crate-type=lib +#![feature(rustc_attrs)] + +#[rustc_force_inline = "the test requires it"] +pub fn forced_with_reason() { +} + +#[rustc_force_inline] +pub fn forced() { +} diff --git a/tests/ui/force-inlining/cast.rs b/tests/ui/force-inlining/cast.rs new file mode 100644 index 0000000000000..8f51ec8397736 --- /dev/null +++ b/tests/ui/force-inlining/cast.rs @@ -0,0 +1,25 @@ +//@ check-fail +//@ compile-flags: --crate-type=lib +#![allow(internal_features)] +#![feature(rustc_attrs)] + +#[rustc_force_inline] +pub fn callee(x: isize) -> usize { unimplemented!() } + +fn a() { + let _: fn(isize) -> usize = callee; +//~^ ERROR cannot coerce functions which must be inlined to function pointers +} + +fn b() { + let _ = callee as fn(isize) -> usize; +//~^ ERROR non-primitive cast +} + +fn c() { + let _: [fn(isize) -> usize; 2] = [ + callee, +//~^ ERROR cannot coerce functions which must be inlined to function pointers + callee, + ]; +} diff --git a/tests/ui/force-inlining/cast.stderr b/tests/ui/force-inlining/cast.stderr new file mode 100644 index 0000000000000..116919e5fe7f0 --- /dev/null +++ b/tests/ui/force-inlining/cast.stderr @@ -0,0 +1,40 @@ +error[E0308]: cannot coerce functions which must be inlined to function pointers + --> $DIR/cast.rs:10:33 + | +LL | let _: fn(isize) -> usize = callee; + | ------------------ ^^^^^^ cannot coerce functions which must be inlined to function pointers + | | + | expected due to this + | + = note: expected fn pointer `fn(_) -> _` + found fn item `fn(_) -> _ {callee}` + = note: fn items are distinct from fn pointers +help: consider casting to a fn pointer + | +LL | let _: fn(isize) -> usize = callee as fn(isize) -> usize; + | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +error[E0605]: non-primitive cast: `fn(isize) -> usize {callee}` as `fn(isize) -> usize` + --> $DIR/cast.rs:15:13 + | +LL | let _ = callee as fn(isize) -> usize; + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ invalid cast + +error[E0308]: cannot coerce functions which must be inlined to function pointers + --> $DIR/cast.rs:21:9 + | +LL | callee, + | ^^^^^^ cannot coerce functions which must be inlined to function pointers + | + = note: expected fn pointer `fn(_) -> _` + found fn item `fn(_) -> _ {callee}` + = note: fn items are distinct from fn pointers +help: consider casting to a fn pointer + | +LL | callee as fn(isize) -> usize, + | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +error: aborting due to 3 previous errors + +Some errors have detailed explanations: E0308, E0605. +For more information about an error, try `rustc --explain E0308`. diff --git a/tests/ui/force-inlining/cross-crate.rs b/tests/ui/force-inlining/cross-crate.rs new file mode 100644 index 0000000000000..a3f0c2c873ea8 --- /dev/null +++ b/tests/ui/force-inlining/cross-crate.rs @@ -0,0 +1,13 @@ +//@ aux-build:callees.rs +//@ build-pass +//@ compile-flags: --crate-type=lib + +extern crate callees; + +// Test that forced inlining across crates works as expected. + +pub fn caller() { + callees::forced(); + + callees::forced_with_reason(); +} diff --git a/tests/ui/force-inlining/deny-async.rs b/tests/ui/force-inlining/deny-async.rs new file mode 100644 index 0000000000000..d5b6dc000f70d --- /dev/null +++ b/tests/ui/force-inlining/deny-async.rs @@ -0,0 +1,25 @@ +//@ check-fail +//@ compile-flags: --crate-type=lib +//@ edition: 2021 +#![allow(internal_features)] +#![feature(rustc_attrs)] + +// Test that forced inlining into async functions w/ errors works as expected. + +#[rustc_no_mir_inline] +#[rustc_force_inline] +pub fn callee() { +} + +#[rustc_no_mir_inline] +#[rustc_force_inline = "the test requires it"] +pub fn callee_justified() { +} + +async fn async_caller() { + callee(); +//~^ ERROR `callee` could not be inlined into `async_caller::{closure#0}` but is required to be inlined + + callee_justified(); +//~^ ERROR `callee_justified` could not be inlined into `async_caller::{closure#0}` but is required to be inlined +} diff --git a/tests/ui/force-inlining/deny-async.stderr b/tests/ui/force-inlining/deny-async.stderr new file mode 100644 index 0000000000000..7b077174ee89e --- /dev/null +++ b/tests/ui/force-inlining/deny-async.stderr @@ -0,0 +1,19 @@ +error: `callee` could not be inlined into `async_caller::{closure#0}` but is required to be inlined + --> $DIR/deny-async.rs:20:5 + | +LL | callee(); + | ^^^^^^^^ ...`callee` called here + | + = note: could not be inlined due to: #[rustc_no_mir_inline] + +error: `callee_justified` could not be inlined into `async_caller::{closure#0}` but is required to be inlined + --> $DIR/deny-async.rs:23:5 + | +LL | callee_justified(); + | ^^^^^^^^^^^^^^^^^^ ...`callee_justified` called here + | + = note: could not be inlined due to: #[rustc_no_mir_inline] + = note: `callee_justified` is required to be inlined to: the test requires it + +error: aborting due to 2 previous errors + diff --git a/tests/ui/force-inlining/deny-closure.rs b/tests/ui/force-inlining/deny-closure.rs new file mode 100644 index 0000000000000..0057f1e78540c --- /dev/null +++ b/tests/ui/force-inlining/deny-closure.rs @@ -0,0 +1,26 @@ +//@ build-fail +//@ compile-flags: --crate-type=lib +#![allow(internal_features)] +#![feature(rustc_attrs)] + +// Test that forced inlining into closures w/ errors works as expected. + +#[rustc_no_mir_inline] +#[rustc_force_inline] +pub fn callee() { +} + +#[rustc_no_mir_inline] +#[rustc_force_inline = "the test requires it"] +pub fn callee_justified() { +} + +pub fn caller() { + (|| { + callee(); +//~^ ERROR `callee` could not be inlined into `caller::{closure#0}` but is required to be inlined + + callee_justified(); +//~^ ERROR `callee_justified` could not be inlined into `caller::{closure#0}` but is required to be inlined + })(); +} diff --git a/tests/ui/force-inlining/deny-closure.stderr b/tests/ui/force-inlining/deny-closure.stderr new file mode 100644 index 0000000000000..9600fa68f540a --- /dev/null +++ b/tests/ui/force-inlining/deny-closure.stderr @@ -0,0 +1,30 @@ +error: `callee` could not be inlined into `caller::{closure#0}` but is required to be inlined + --> $DIR/deny-closure.rs:20:9 + | +LL | callee(); + | ^^^^^^^^ ...`callee` called here + | + = note: could not be inlined due to: #[rustc_no_mir_inline] + +error: `callee_justified` could not be inlined into `caller::{closure#0}` but is required to be inlined + --> $DIR/deny-closure.rs:23:9 + | +LL | callee_justified(); + | ^^^^^^^^^^^^^^^^^^ ...`callee_justified` called here + | + = note: could not be inlined due to: #[rustc_no_mir_inline] + = note: `callee_justified` is required to be inlined to: the test requires it + +note: the above error was encountered while instantiating `fn caller::{closure#0}` + --> $DIR/deny-closure.rs:19:5 + | +LL | / (|| { +LL | | callee(); +LL | | +... | +LL | | +LL | | })(); + | |________^ + +error: aborting due to 2 previous errors + diff --git a/tests/ui/force-inlining/deny.rs b/tests/ui/force-inlining/deny.rs new file mode 100644 index 0000000000000..a19e820cff28d --- /dev/null +++ b/tests/ui/force-inlining/deny.rs @@ -0,0 +1,24 @@ +//@ build-fail +//@ compile-flags: --crate-type=lib +#![allow(internal_features)] +#![feature(rustc_attrs)] + +// Test that forced inlining w/ errors works as expected. + +#[rustc_no_mir_inline] +#[rustc_force_inline] +pub fn callee() { +} + +#[rustc_no_mir_inline] +#[rustc_force_inline = "the test requires it"] +pub fn callee_justified() { +} + +pub fn caller() { + callee(); +//~^ ERROR `callee` could not be inlined into `caller` but is required to be inlined + + callee_justified(); +//~^ ERROR `callee_justified` could not be inlined into `caller` but is required to be inlined +} diff --git a/tests/ui/force-inlining/deny.stderr b/tests/ui/force-inlining/deny.stderr new file mode 100644 index 0000000000000..3983bead182eb --- /dev/null +++ b/tests/ui/force-inlining/deny.stderr @@ -0,0 +1,19 @@ +error: `callee` could not be inlined into `caller` but is required to be inlined + --> $DIR/deny.rs:19:5 + | +LL | callee(); + | ^^^^^^^^ ...`callee` called here + | + = note: could not be inlined due to: #[rustc_no_mir_inline] + +error: `callee_justified` could not be inlined into `caller` but is required to be inlined + --> $DIR/deny.rs:22:5 + | +LL | callee_justified(); + | ^^^^^^^^^^^^^^^^^^ ...`callee_justified` called here + | + = note: could not be inlined due to: #[rustc_no_mir_inline] + = note: `callee_justified` is required to be inlined to: the test requires it + +error: aborting due to 2 previous errors + diff --git a/tests/ui/force-inlining/gate.rs b/tests/ui/force-inlining/gate.rs new file mode 100644 index 0000000000000..d6a01a74a44b5 --- /dev/null +++ b/tests/ui/force-inlining/gate.rs @@ -0,0 +1,12 @@ +//@ compile-flags: --crate-type=lib +#![allow(internal_features)] + +#[rustc_force_inline] +//~^ ERROR #![rustc_force_inline] forces a free function to be inlined +pub fn bare() { +} + +#[rustc_force_inline = "the test requires it"] +//~^ ERROR #![rustc_force_inline] forces a free function to be inlined +pub fn justified() { +} diff --git a/tests/ui/force-inlining/gate.stderr b/tests/ui/force-inlining/gate.stderr new file mode 100644 index 0000000000000..e3973a08c232b --- /dev/null +++ b/tests/ui/force-inlining/gate.stderr @@ -0,0 +1,21 @@ +error[E0658]: #![rustc_force_inline] forces a free function to be inlined + --> $DIR/gate.rs:4:1 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ + | + = help: add `#![feature(rustc_attrs)]` to the crate attributes to enable + = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date + +error[E0658]: #![rustc_force_inline] forces a free function to be inlined + --> $DIR/gate.rs:9:1 + | +LL | #[rustc_force_inline = "the test requires it"] + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | + = help: add `#![feature(rustc_attrs)]` to the crate attributes to enable + = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date + +error: aborting due to 2 previous errors + +For more information about this error, try `rustc --explain E0658`. diff --git a/tests/ui/force-inlining/invalid.rs b/tests/ui/force-inlining/invalid.rs new file mode 100644 index 0000000000000..7574078b245c2 --- /dev/null +++ b/tests/ui/force-inlining/invalid.rs @@ -0,0 +1,164 @@ +//@ edition: 2024 +#![allow(internal_features, unused_imports, unused_macros)] +#![feature(extern_types)] +#![feature(gen_blocks)] +#![feature(rustc_attrs)] +#![feature(stmt_expr_attributes)] +#![feature(trait_alias)] + +// Test that invalid force inlining attributes error as expected. + +#[rustc_force_inline("foo")] +//~^ ERROR malformed `rustc_force_inline` attribute input +pub fn forced1() { +} + +#[rustc_force_inline(bar, baz)] +//~^ ERROR malformed `rustc_force_inline` attribute input +pub fn forced2() { +} + +#[rustc_force_inline(2)] +//~^ ERROR malformed `rustc_force_inline` attribute input +pub fn forced3() { +} + +#[rustc_force_inline = 2] +//~^ ERROR malformed `rustc_force_inline` attribute input +pub fn forced4() { +} + +#[rustc_force_inline] +//~^ ERROR attribute should be applied to a function +extern crate std as other_std; + +#[rustc_force_inline] +//~^ ERROR attribute should be applied to a function +use std::collections::HashMap; + +#[rustc_force_inline] +//~^ ERROR attribute should be applied to a function +static _FOO: &'static str = "FOO"; + +#[rustc_force_inline] +//~^ ERROR attribute should be applied to a function +const _BAR: u32 = 3; + +#[rustc_force_inline] +//~^ ERROR attribute should be applied to a function +mod foo { } + +#[rustc_force_inline] +//~^ ERROR attribute should be applied to a function +unsafe extern "C" { + #[rustc_force_inline] +//~^ ERROR attribute should be applied to a function + static X: &'static u32; + + #[rustc_force_inline] +//~^ ERROR attribute should be applied to a function + type Y; + + #[rustc_force_inline] +//~^ ERROR attribute should be applied to a function + fn foo(); +} + +#[rustc_force_inline] +//~^ ERROR attribute should be applied to a function +type Foo = u32; + +#[rustc_force_inline] +//~^ ERROR attribute should be applied to a function +enum Bar<#[rustc_force_inline] T> { +//~^ ERROR attribute should be applied to a function + #[rustc_force_inline] +//~^ ERROR attribute should be applied to a function + Baz(std::marker::PhantomData), +} + +#[rustc_force_inline] +//~^ ERROR attribute should be applied to a function +struct Qux { + #[rustc_force_inline] +//~^ ERROR attribute should be applied to a function + field: u32, +} + +#[rustc_force_inline] +//~^ ERROR attribute should be applied to a function +union FooBar { + x: u32, + y: u32, +} + +#[rustc_force_inline] +//~^ ERROR attribute should be applied to a function +trait FooBaz { + #[rustc_force_inline] +//~^ ERROR attribute should be applied to a function + type Foo; + #[rustc_force_inline] +//~^ ERROR attribute should be applied to a function + const Bar: i32; + + #[rustc_force_inline] +//~^ ERROR attribute should be applied to a function + fn foo() {} +} + +#[rustc_force_inline] +//~^ ERROR attribute should be applied to a function +trait FooQux = FooBaz; + +#[rustc_force_inline] +//~^ ERROR attribute should be applied to a function +impl Bar { + #[rustc_force_inline] +//~^ ERROR attribute should be applied to a function + fn foo() {} +} + +#[rustc_force_inline] +//~^ ERROR attribute should be applied to a function +impl FooBaz for Bar { + type Foo = u32; + const Bar: i32 = 3; +} + +#[rustc_force_inline] +//~^ ERROR attribute should be applied to a function +macro_rules! barqux { ($foo:tt) => { $foo }; } + +fn barqux(#[rustc_force_inline] _x: u32) {} +//~^ ERROR allow, cfg, cfg_attr, deny, expect, forbid, and warn are the only allowed built-in attributes in function parameters +//~^^ ERROR attribute should be applied to a function + +#[rustc_force_inline] +//~^ ERROR attribute cannot be applied to a `async`, `gen` or `async gen` function +async fn async_foo() {} + +#[rustc_force_inline] +//~^ ERROR attribute cannot be applied to a `async`, `gen` or `async gen` function +gen fn gen_foo() {} + +#[rustc_force_inline] +//~^ ERROR attribute cannot be applied to a `async`, `gen` or `async gen` function +async gen fn async_gen_foo() {} + +fn main() { + let _x = #[rustc_force_inline] || { }; +//~^ ERROR attribute should be applied to a function + let _y = #[rustc_force_inline] 3 + 4; +//~^ ERROR attribute should be applied to a function + #[rustc_force_inline] +//~^ ERROR attribute should be applied to a function + let _z = 3; + + match _z { + #[rustc_force_inline] +//~^ ERROR attribute should be applied to a function + 1 => (), + _ => (), + } +} diff --git a/tests/ui/force-inlining/invalid.stderr b/tests/ui/force-inlining/invalid.stderr new file mode 100644 index 0000000000000..36d824fd05fc7 --- /dev/null +++ b/tests/ui/force-inlining/invalid.stderr @@ -0,0 +1,378 @@ +error: malformed `rustc_force_inline` attribute input + --> $DIR/invalid.rs:11:1 + | +LL | #[rustc_force_inline("foo")] + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | +help: the following are the possible correct uses + | +LL | #[rustc_force_inline = "reason"] + | +LL | #[rustc_force_inline] + | + +error: malformed `rustc_force_inline` attribute input + --> $DIR/invalid.rs:16:1 + | +LL | #[rustc_force_inline(bar, baz)] + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | +help: the following are the possible correct uses + | +LL | #[rustc_force_inline = "reason"] + | +LL | #[rustc_force_inline] + | + +error: malformed `rustc_force_inline` attribute input + --> $DIR/invalid.rs:21:1 + | +LL | #[rustc_force_inline(2)] + | ^^^^^^^^^^^^^^^^^^^^^^^^ + | +help: the following are the possible correct uses + | +LL | #[rustc_force_inline = "reason"] + | +LL | #[rustc_force_inline] + | + +error: malformed `rustc_force_inline` attribute input + --> $DIR/invalid.rs:26:1 + | +LL | #[rustc_force_inline = 2] + | ^^^^^^^^^^^^^^^^^^^^^^^^^ + | +help: the following are the possible correct uses + | +LL | #[rustc_force_inline = "reason"] + | +LL | #[rustc_force_inline] + | + +error: allow, cfg, cfg_attr, deny, expect, forbid, and warn are the only allowed built-in attributes in function parameters + --> $DIR/invalid.rs:133:11 + | +LL | fn barqux(#[rustc_force_inline] _x: u32) {} + | ^^^^^^^^^^^^^^^^^^^^^ + +error: attribute should be applied to a function + --> $DIR/invalid.rs:31:1 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | extern crate std as other_std; + | ------------------------------ not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:35:1 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | use std::collections::HashMap; + | ------------------------------ not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:39:1 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | static _FOO: &'static str = "FOO"; + | ---------------------------------- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:43:1 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | const _BAR: u32 = 3; + | -------------------- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:47:1 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | mod foo { } + | ----------- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:51:1 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | / unsafe extern "C" { +LL | | #[rustc_force_inline] +LL | | +LL | | static X: &'static u32; +... | +LL | | fn foo(); +LL | | } + | |_- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:67:1 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | type Foo = u32; + | --------------- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:71:1 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | / enum Bar<#[rustc_force_inline] T> { +LL | | +LL | | #[rustc_force_inline] +LL | | +LL | | Baz(std::marker::PhantomData), +LL | | } + | |_- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:73:10 + | +LL | enum Bar<#[rustc_force_inline] T> { + | ^^^^^^^^^^^^^^^^^^^^^ - not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:75:5 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | Baz(std::marker::PhantomData), + | -------------------------------- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:80:1 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | / struct Qux { +LL | | #[rustc_force_inline] +LL | | +LL | | field: u32, +LL | | } + | |_- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:83:5 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | field: u32, + | ---------- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:88:1 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | / union FooBar { +LL | | x: u32, +LL | | y: u32, +LL | | } + | |_- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:95:1 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | / trait FooBaz { +LL | | #[rustc_force_inline] +LL | | +LL | | type Foo; +... | +LL | | fn foo() {} +LL | | } + | |_- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:110:1 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | trait FooQux = FooBaz; + | ---------------------- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:114:1 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | / impl Bar { +LL | | #[rustc_force_inline] +LL | | +LL | | fn foo() {} +LL | | } + | |_- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:122:1 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | / impl FooBaz for Bar { +LL | | type Foo = u32; +LL | | const Bar: i32 = 3; +LL | | } + | |_- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:129:1 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | macro_rules! barqux { ($foo:tt) => { $foo }; } + | ---------------------------------------------- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:133:11 + | +LL | fn barqux(#[rustc_force_inline] _x: u32) {} + | ^^^^^^^^^^^^^^^^^^^^^-------- + | | + | not a function definition + +error: attribute cannot be applied to a `async`, `gen` or `async gen` function + --> $DIR/invalid.rs:137:1 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | async fn async_foo() {} + | -------------------- `async`, `gen` or `async gen` function + +error: attribute cannot be applied to a `async`, `gen` or `async gen` function + --> $DIR/invalid.rs:141:1 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | gen fn gen_foo() {} + | ---------------- `async`, `gen` or `async gen` function + +error: attribute cannot be applied to a `async`, `gen` or `async gen` function + --> $DIR/invalid.rs:145:1 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | async gen fn async_gen_foo() {} + | ---------------------------- `async`, `gen` or `async gen` function + +error: attribute should be applied to a function + --> $DIR/invalid.rs:150:14 + | +LL | let _x = #[rustc_force_inline] || { }; + | ^^^^^^^^^^^^^^^^^^^^^ ------ not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:152:14 + | +LL | let _y = #[rustc_force_inline] 3 + 4; + | ^^^^^^^^^^^^^^^^^^^^^ - not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:154:5 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | let _z = 3; + | ----------- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:159:9 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | 1 => (), + | ------- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:98:5 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | type Foo; + | --------- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:101:5 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | const Bar: i32; + | --------------- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:105:5 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | fn foo() {} + | ----------- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:117:5 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | fn foo() {} + | ----------- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:54:5 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | static X: &'static u32; + | ----------------------- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:58:5 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | type Y; + | ------- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:62:5 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | fn foo(); + | --------- not a function definition + +error: aborting due to 38 previous errors + From d370964ec196113e3725e2f0599bc929af869483 Mon Sep 17 00:00:00 2001 From: David Wood Date: Tue, 10 Dec 2024 12:05:24 +0000 Subject: [PATCH 2/6] codegen_attrs: force inlining takes precedence --- compiler/rustc_codegen_ssa/src/codegen_attrs.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs index 3b14d47f4772c..2bf4bec2e1081 100644 --- a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs +++ b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs @@ -552,7 +552,12 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { Some(MetaItemKind::NameValue(_)) => ia, None => ia, } - } else if attr.has_name(sym::rustc_force_inline) && tcx.features().rustc_attrs() { + } else { + ia + } + }); + codegen_fn_attrs.inline = attrs.iter().fold(codegen_fn_attrs.inline, |ia, attr| { + if attr.has_name(sym::rustc_force_inline) && tcx.features().rustc_attrs() { match attr.meta_kind() { Some(MetaItemKind::NameValue(lit)) => { InlineAttr::Force { attr_span: attr.span, reason: Some(lit.symbol) } From 26659c6b4dd64404aef3ad199030665269e1e153 Mon Sep 17 00:00:00 2001 From: David Wood Date: Tue, 10 Dec 2024 12:06:24 +0000 Subject: [PATCH 3/6] inline: force inlining shims --- compiler/rustc_mir_transform/src/inline.rs | 26 +++++++++++++--------- compiler/rustc_mir_transform/src/shim.rs | 4 +++- tests/ui/force-inlining/shims.rs | 9 ++++++++ 3 files changed, 27 insertions(+), 12 deletions(-) create mode 100644 tests/ui/force-inlining/shims.rs diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs index 424e774b3a1c7..518d4925baa95 100644 --- a/compiler/rustc_mir_transform/src/inline.rs +++ b/compiler/rustc_mir_transform/src/inline.rs @@ -6,7 +6,7 @@ use std::ops::{Range, RangeFrom}; use rustc_abi::{ExternAbi, FieldIdx}; use rustc_attr::InlineAttr; use rustc_hir::def::DefKind; -use rustc_hir::def_id::{DefId, LocalDefId}; +use rustc_hir::def_id::DefId; use rustc_index::Idx; use rustc_index::bit_set::BitSet; use rustc_middle::bug; @@ -98,12 +98,12 @@ impl<'tcx> crate::MirPass<'tcx> for ForceInline { } trait Inliner<'tcx> { - fn new(tcx: TyCtxt<'tcx>, def_id: LocalDefId, body: &Body<'tcx>) -> Self; + fn new(tcx: TyCtxt<'tcx>, def_id: DefId, body: &Body<'tcx>) -> Self; fn tcx(&self) -> TyCtxt<'tcx>; fn typing_env(&self) -> ty::TypingEnv<'tcx>; fn history(&self) -> &[DefId]; - fn caller_def_id(&self) -> LocalDefId; + fn caller_def_id(&self) -> DefId; /// Has the caller body been changed? fn changed(self) -> bool; @@ -146,7 +146,7 @@ struct ForceInliner<'tcx> { tcx: TyCtxt<'tcx>, typing_env: ty::TypingEnv<'tcx>, /// `DefId` of caller. - def_id: LocalDefId, + def_id: DefId, /// Stack of inlined instances. /// We only check the `DefId` and not the args because we want to /// avoid inlining cases of polymorphic recursion. @@ -158,7 +158,7 @@ struct ForceInliner<'tcx> { } impl<'tcx> Inliner<'tcx> for ForceInliner<'tcx> { - fn new(tcx: TyCtxt<'tcx>, def_id: LocalDefId, body: &Body<'tcx>) -> Self { + fn new(tcx: TyCtxt<'tcx>, def_id: DefId, body: &Body<'tcx>) -> Self { Self { tcx, typing_env: body.typing_env(tcx), def_id, history: Vec::new(), changed: false } } @@ -174,7 +174,7 @@ impl<'tcx> Inliner<'tcx> for ForceInliner<'tcx> { &self.history } - fn caller_def_id(&self) -> LocalDefId { + fn caller_def_id(&self) -> DefId { self.def_id } @@ -248,7 +248,7 @@ struct NormalInliner<'tcx> { tcx: TyCtxt<'tcx>, typing_env: ty::TypingEnv<'tcx>, /// `DefId` of caller. - def_id: LocalDefId, + def_id: DefId, /// Stack of inlined instances. /// We only check the `DefId` and not the args because we want to /// avoid inlining cases of polymorphic recursion. @@ -263,7 +263,7 @@ struct NormalInliner<'tcx> { } impl<'tcx> Inliner<'tcx> for NormalInliner<'tcx> { - fn new(tcx: TyCtxt<'tcx>, def_id: LocalDefId, body: &Body<'tcx>) -> Self { + fn new(tcx: TyCtxt<'tcx>, def_id: DefId, body: &Body<'tcx>) -> Self { let typing_env = body.typing_env(tcx); let codegen_fn_attrs = tcx.codegen_fn_attrs(def_id); @@ -284,7 +284,7 @@ impl<'tcx> Inliner<'tcx> for NormalInliner<'tcx> { self.tcx } - fn caller_def_id(&self) -> LocalDefId { + fn caller_def_id(&self) -> DefId { self.def_id } @@ -442,7 +442,7 @@ impl<'tcx> Inliner<'tcx> for NormalInliner<'tcx> { } fn inline<'tcx, T: Inliner<'tcx>>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> bool { - let def_id = body.source.def_id().expect_local(); + let def_id = body.source.def_id(); // Only do inlining into fn bodies. if !tcx.hir().body_owner_kind(def_id).is_fn_or_closure() { @@ -722,7 +722,11 @@ fn check_mir_is_available<'tcx, I: Inliner<'tcx>>( return Ok(()); } - if callee_def_id.is_local() { + if callee_def_id.is_local() + && !inliner + .tcx() + .is_lang_item(inliner.tcx().parent(caller_def_id), rustc_hir::LangItem::FnOnce) + { // If we know for sure that the function we're calling will itself try to // call us, then we avoid inlining that function. if inliner.tcx().mir_callgraph_reachable((callee, caller_def_id.expect_local())) { diff --git a/compiler/rustc_mir_transform/src/shim.rs b/compiler/rustc_mir_transform/src/shim.rs index b8383e734e2ba..a707ea760e520 100644 --- a/compiler/rustc_mir_transform/src/shim.rs +++ b/compiler/rustc_mir_transform/src/shim.rs @@ -19,7 +19,7 @@ use rustc_span::{DUMMY_SP, Span}; use tracing::{debug, instrument}; use crate::{ - abort_unwinding_calls, add_call_guards, add_moves_for_packed_drops, deref_separator, + abort_unwinding_calls, add_call_guards, add_moves_for_packed_drops, deref_separator, inline, instsimplify, mentioned_items, pass_manager as pm, remove_noop_landing_pads, simplify, }; @@ -154,6 +154,8 @@ fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceKind<'tcx>) -> Body< &remove_noop_landing_pads::RemoveNoopLandingPads, &simplify::SimplifyCfg::MakeShim, &instsimplify::InstSimplify::BeforeInline, + // Perform inlining of `#[rustc_force_inline]`-annotated callees. + &inline::ForceInline, &abort_unwinding_calls::AbortUnwindingCalls, &add_call_guards::CriticalCallEdges, ], diff --git a/tests/ui/force-inlining/shims.rs b/tests/ui/force-inlining/shims.rs new file mode 100644 index 0000000000000..03b7d07cda39e --- /dev/null +++ b/tests/ui/force-inlining/shims.rs @@ -0,0 +1,9 @@ +//@ build-pass +#![allow(internal_features)] +#![feature(rustc_attrs)] + +#[rustc_force_inline] +fn f() {} +fn g(t: T) { t(); } + +fn main() { g(f); } From f7640c938bbaf17bd235aba379171e42231afe00 Mon Sep 17 00:00:00 2001 From: David Wood Date: Tue, 10 Dec 2024 12:06:57 +0000 Subject: [PATCH 4/6] inline: re-introduce some callee body checks --- compiler/rustc_mir_transform/src/inline.rs | 31 +++++++++- tests/ui/force-inlining/asm.rs | 68 ++++++++++++++++++++++ tests/ui/force-inlining/asm.stderr | 34 +++++++++++ 3 files changed, 131 insertions(+), 2 deletions(-) create mode 100644 tests/ui/force-inlining/asm.rs create mode 100644 tests/ui/force-inlining/asm.stderr diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs index 518d4925baa95..621dba06694f3 100644 --- a/compiler/rustc_mir_transform/src/inline.rs +++ b/compiler/rustc_mir_transform/src/inline.rs @@ -195,10 +195,37 @@ impl<'tcx> Inliner<'tcx> for ForceInliner<'tcx> { &self, _: &CallSite<'tcx>, callee_body: &Body<'tcx>, - _: &CodegenFnAttrs, + callee_attrs: &CodegenFnAttrs, _: bool, ) -> Result<(), &'static str> { - if let Some(_) = callee_body.tainted_by_errors { Err("body has errors") } else { Ok(()) } + if callee_body.tainted_by_errors.is_some() { + return Err("body has errors"); + } + + let caller_attrs = self.tcx().codegen_fn_attrs(self.caller_def_id()); + if callee_attrs.instruction_set != caller_attrs.instruction_set + && callee_body + .basic_blocks + .iter() + .any(|bb| matches!(bb.terminator().kind, TerminatorKind::InlineAsm { .. })) + { + // During the attribute checking stage we allow a callee with no + // instruction_set assigned to count as compatible with a function that does + // assign one. However, during this stage we require an exact match when any + // inline-asm is detected. LLVM will still possibly do an inline later on + // if the no-attribute function ends up with the same instruction set anyway. + Err("cannot move inline-asm across instruction sets") + } else if callee_body + .basic_blocks + .iter() + .any(|bb| matches!(bb.terminator().kind, TerminatorKind::TailCall { .. })) + { + // FIXME(explicit_tail_calls): figure out how exactly functions containing tail + // calls can be inlined (and if they even should) + Err("can't inline functions with tail calls") + } else { + Ok(()) + } } fn inline_limit_for_block(&self) -> Option { diff --git a/tests/ui/force-inlining/asm.rs b/tests/ui/force-inlining/asm.rs new file mode 100644 index 0000000000000..85014ffa51588 --- /dev/null +++ b/tests/ui/force-inlining/asm.rs @@ -0,0 +1,68 @@ +//@ build-fail +//@ compile-flags: --crate-type=lib --target thumbv4t-none-eabi +//@ needs-llvm-components: arm + +// Checks that forced inlining won't mix asm with incompatible instruction sets. + +#![crate_type = "lib"] +#![feature(rustc_attrs)] +#![feature(no_core, lang_items)] +#![no_core] + +#[lang = "sized"] +pub trait Sized {} +#[lang = "copy"] +pub trait Copy {} +#[lang = "freeze"] +pub unsafe trait Freeze {} + +#[lang = "start"] +fn start(_main: fn() -> T, _argc: isize, _argv: *const *const u8, _sigpipe: u8) -> isize { + 0 +} + +#[rustc_builtin_macro] +#[macro_export] +macro_rules! asm { + ("assembly template", + $(operands,)* + $(options($(option),*))? + ) => { + /* compiler built-in */ + }; +} + +#[instruction_set(arm::a32)] +#[rustc_force_inline] +fn instruction_set_a32() {} + +#[instruction_set(arm::t32)] +#[rustc_force_inline] +fn instruction_set_t32() {} + +#[rustc_force_inline] +fn instruction_set_default() {} + +#[rustc_force_inline] +fn inline_always_and_using_inline_asm() { + unsafe { asm!("/* do nothing */") }; +} + +#[instruction_set(arm::t32)] +pub fn t32() { + instruction_set_a32(); +//~^ ERROR `instruction_set_a32` could not be inlined into `t32` but is required to be inlined + instruction_set_t32(); + instruction_set_default(); + inline_always_and_using_inline_asm(); +//~^ ERROR `inline_always_and_using_inline_asm` could not be inlined into `t32` but is required to be inlined +} + +pub fn default() { + instruction_set_a32(); +//~^ ERROR `instruction_set_a32` could not be inlined into `default` but is required to be inlined + instruction_set_t32(); +//~^ ERROR `instruction_set_t32` could not be inlined into `default` but is required to be inlined + instruction_set_default(); + inline_always_and_using_inline_asm(); +} diff --git a/tests/ui/force-inlining/asm.stderr b/tests/ui/force-inlining/asm.stderr new file mode 100644 index 0000000000000..ef04688965bb3 --- /dev/null +++ b/tests/ui/force-inlining/asm.stderr @@ -0,0 +1,34 @@ +error: `instruction_set_a32` could not be inlined into `t32` but is required to be inlined + --> $DIR/asm.rs:53:5 + | +LL | instruction_set_a32(); + | ^^^^^^^^^^^^^^^^^^^^^ ...`instruction_set_a32` called here + | + = note: could not be inlined due to: incompatible instruction set + +error: `inline_always_and_using_inline_asm` could not be inlined into `t32` but is required to be inlined + --> $DIR/asm.rs:57:5 + | +LL | inline_always_and_using_inline_asm(); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ...`inline_always_and_using_inline_asm` called here + | + = note: could not be inlined due to: cannot move inline-asm across instruction sets + +error: `instruction_set_a32` could not be inlined into `default` but is required to be inlined + --> $DIR/asm.rs:62:5 + | +LL | instruction_set_a32(); + | ^^^^^^^^^^^^^^^^^^^^^ ...`instruction_set_a32` called here + | + = note: could not be inlined due to: incompatible instruction set + +error: `instruction_set_t32` could not be inlined into `default` but is required to be inlined + --> $DIR/asm.rs:64:5 + | +LL | instruction_set_t32(); + | ^^^^^^^^^^^^^^^^^^^^^ ...`instruction_set_t32` called here + | + = note: could not be inlined due to: incompatible instruction set + +error: aborting due to 4 previous errors + From da261aa7310ac029bf5009bc89688b7870e21a98 Mon Sep 17 00:00:00 2001 From: David Wood Date: Tue, 10 Dec 2024 12:07:20 +0000 Subject: [PATCH 5/6] inline: remove unnecessary promoted check --- compiler/rustc_mir_transform/src/inline.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs index 621dba06694f3..6e02cc704feb1 100644 --- a/compiler/rustc_mir_transform/src/inline.rs +++ b/compiler/rustc_mir_transform/src/inline.rs @@ -332,10 +332,6 @@ impl<'tcx> Inliner<'tcx> for NormalInliner<'tcx> { } fn check_caller_mir_body(&self, body: &Body<'tcx>) -> bool { - if body.source.promoted.is_some() { - return false; - } - // Avoid inlining into coroutines, since their `optimized_mir` is used for layout computation, // which can create a cycle, even when no attempt is made to inline the function in the other // direction. From 08f365c66a40c0738728bd498073cdb1823ff781 Mon Sep 17 00:00:00 2001 From: David Wood Date: Tue, 10 Dec 2024 12:07:40 +0000 Subject: [PATCH 6/6] validator: move force inline check --- compiler/rustc_mir_transform/src/inline.rs | 10 +++--- compiler/rustc_mir_transform/src/validate.rs | 33 ++++++-------------- 2 files changed, 15 insertions(+), 28 deletions(-) diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs index 6e02cc704feb1..b818fe630c1d9 100644 --- a/compiler/rustc_mir_transform/src/inline.rs +++ b/compiler/rustc_mir_transform/src/inline.rs @@ -502,11 +502,6 @@ fn process_blocks<'tcx, I: Inliner<'tcx>>( let span = trace_span!("process_blocks", %callsite.callee, ?bb); let _guard = span.enter(); - if !inliner.should_inline_for_callee(callsite.callee.def_id()) { - debug!("not enabled"); - continue; - } - match try_inlining(inliner, caller_body, &callsite) { Err(reason) => { debug!("not-inlined {} [{}]", callsite.callee, reason); @@ -541,6 +536,11 @@ fn resolve_callsite<'tcx, I: Inliner<'tcx>>( if let TerminatorKind::Call { ref func, fn_span, .. } = terminator.kind { let func_ty = func.ty(caller_body, tcx); if let ty::FnDef(def_id, args) = *func_ty.kind() { + if !inliner.should_inline_for_callee(def_id) { + debug!("not enabled"); + return None; + } + // To resolve an instance its args have to be fully normalized. let args = tcx.try_normalize_erasing_regions(inliner.typing_env(), args).ok()?; let callee = diff --git a/compiler/rustc_mir_transform/src/validate.rs b/compiler/rustc_mir_transform/src/validate.rs index 927cbe998527e..f9803cd56ce8c 100644 --- a/compiler/rustc_mir_transform/src/validate.rs +++ b/compiler/rustc_mir_transform/src/validate.rs @@ -80,7 +80,7 @@ impl<'tcx> crate::MirPass<'tcx> for Validator { cfg_checker.fail(location, msg); } - if let MirPhase::Runtime(phase) = body.phase { + if let MirPhase::Runtime(_) = body.phase { if let ty::InstanceKind::Item(_) = body.source.instance { if body.has_free_regions() { cfg_checker.fail( @@ -89,27 +89,6 @@ impl<'tcx> crate::MirPass<'tcx> for Validator { ); } } - - if phase >= RuntimePhase::Optimized - && body - .basic_blocks - .iter() - .filter_map(|bb| match &bb.terminator().kind { - TerminatorKind::Call { func, .. } - | TerminatorKind::TailCall { func, .. } => Some(func), - _ => None, - }) - .filter_map(|func| match func.ty(&body.local_decls, tcx).kind() { - ty::FnDef(did, ..) => Some(did), - _ => None, - }) - .any(|did| matches!(tcx.codegen_fn_attrs(did).inline, InlineAttr::Force { .. })) - { - cfg_checker.fail( - Location::START, - "`#[rustc_force_inline]`-annotated function not inlined", - ); - } } } } @@ -388,7 +367,8 @@ impl<'a, 'tcx> Visitor<'tcx> for CfgChecker<'a, 'tcx> { self.check_edge(location, *target, EdgeKind::Normal); self.check_unwind_edge(location, *unwind); } - TerminatorKind::Call { args, .. } | TerminatorKind::TailCall { args, .. } => { + TerminatorKind::Call { func, args, .. } + | TerminatorKind::TailCall { func, args, .. } => { // FIXME(explicit_tail_calls): refactor this & add tail-call specific checks if let TerminatorKind::Call { target, unwind, destination, .. } = terminator.kind { if let Some(target) = target { @@ -441,6 +421,13 @@ impl<'a, 'tcx> Visitor<'tcx> for CfgChecker<'a, 'tcx> { } } } + + if let ty::FnDef(did, ..) = func.ty(&self.body.local_decls, self.tcx).kind() + && self.body.phase >= MirPhase::Runtime(RuntimePhase::Optimized) + && matches!(self.tcx.codegen_fn_attrs(did).inline, InlineAttr::Force { .. }) + { + self.fail(location, "`#[rustc_force_inline]`-annotated function not inlined"); + } } TerminatorKind::Assert { target, unwind, .. } => { self.check_edge(location, *target, EdgeKind::Normal);