diff --git a/compiler/rustc_attr/src/builtin.rs b/compiler/rustc_attr/src/builtin.rs index 94f9727eb7fbe..2e4d192703c11 100644 --- a/compiler/rustc_attr/src/builtin.rs +++ b/compiler/rustc_attr/src/builtin.rs @@ -47,6 +47,22 @@ pub enum InlineAttr { Hint, Always, Never, + /// `#[rustc_force_inline]` forces inlining to happen in the MIR inliner - it reports an error + /// if the inlining cannot happen. It is limited to only free functions so that the calls + /// can always be resolved. + Force { + attr_span: Span, + reason: Option, + }, +} + +impl InlineAttr { + pub fn always(&self) -> bool { + match self { + InlineAttr::Always | InlineAttr::Force { .. } => true, + InlineAttr::None | InlineAttr::Hint | InlineAttr::Never => false, + } + } } #[derive(Clone, Encodable, Decodable, Debug, PartialEq, Eq, HashStable_Generic)] diff --git a/compiler/rustc_codegen_gcc/src/attributes.rs b/compiler/rustc_codegen_gcc/src/attributes.rs index d20e13e15b944..c106333dbf377 100644 --- a/compiler/rustc_codegen_gcc/src/attributes.rs +++ b/compiler/rustc_codegen_gcc/src/attributes.rs @@ -20,7 +20,7 @@ fn inline_attr<'gcc, 'tcx>( ) -> Option> { match inline { InlineAttr::Hint => Some(FnAttribute::Inline), - InlineAttr::Always => Some(FnAttribute::AlwaysInline), + InlineAttr::Always | InlineAttr::Force { .. } => Some(FnAttribute::AlwaysInline), InlineAttr::Never => { if cx.sess().target.arch != "amdgpu" { Some(FnAttribute::NoInline) diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs index 5552a2410601d..259d9a2080f55 100644 --- a/compiler/rustc_codegen_llvm/src/attributes.rs +++ b/compiler/rustc_codegen_llvm/src/attributes.rs @@ -37,7 +37,9 @@ fn inline_attr<'ll>(cx: &CodegenCx<'ll, '_>, inline: InlineAttr) -> Option<&'ll } match inline { InlineAttr::Hint => Some(AttributeKind::InlineHint.create_attr(cx.llcx)), - InlineAttr::Always => Some(AttributeKind::AlwaysInline.create_attr(cx.llcx)), + InlineAttr::Always | InlineAttr::Force { .. } => { + Some(AttributeKind::AlwaysInline.create_attr(cx.llcx)) + } InlineAttr::Never => { if cx.sess().target.arch != "amdgpu" { Some(AttributeKind::NoInline.create_attr(cx.llcx)) diff --git a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs index ab2d24e8d2dd7..2bf4bec2e1081 100644 --- a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs +++ b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs @@ -18,6 +18,7 @@ use rustc_session::{Session, lint}; use rustc_span::symbol::Ident; use rustc_span::{Span, sym}; use rustc_target::spec::{SanitizerSet, abi}; +use tracing::debug; use crate::errors; use crate::target_features::{check_target_feature_trait_unsafe, from_target_feature_attr}; @@ -522,31 +523,55 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { mixed_export_name_no_mangle_lint_state.lint_if_mixed(tcx); codegen_fn_attrs.inline = attrs.iter().fold(InlineAttr::None, |ia, attr| { - if !attr.has_name(sym::inline) { - return ia; - } - match attr.meta_kind() { - Some(MetaItemKind::Word) => InlineAttr::Hint, - Some(MetaItemKind::List(ref items)) => { - inline_span = Some(attr.span); - if items.len() != 1 { - struct_span_code_err!(tcx.dcx(), attr.span, E0534, "expected one argument") - .emit(); - InlineAttr::None - } else if list_contains_name(items, sym::always) { - InlineAttr::Always - } else if list_contains_name(items, sym::never) { - InlineAttr::Never - } else { - struct_span_code_err!(tcx.dcx(), items[0].span(), E0535, "invalid argument") + if attr.has_name(sym::inline) { + match attr.meta_kind() { + Some(MetaItemKind::Word) => InlineAttr::Hint, + Some(MetaItemKind::List(ref items)) => { + inline_span = Some(attr.span); + if items.len() != 1 { + struct_span_code_err!(tcx.dcx(), attr.span, E0534, "expected one argument") + .emit(); + InlineAttr::None + } else if list_contains_name(items, sym::always) { + InlineAttr::Always + } else if list_contains_name(items, sym::never) { + InlineAttr::Never + } else { + struct_span_code_err!( + tcx.dcx(), + items[0].span(), + E0535, + "invalid argument" + ) .with_help("valid inline arguments are `always` and `never`") .emit(); - InlineAttr::None + InlineAttr::None + } } + Some(MetaItemKind::NameValue(_)) => ia, + None => ia, } - Some(MetaItemKind::NameValue(_)) => ia, - None => ia, + } else { + ia + } + }); + codegen_fn_attrs.inline = attrs.iter().fold(codegen_fn_attrs.inline, |ia, attr| { + if attr.has_name(sym::rustc_force_inline) && tcx.features().rustc_attrs() { + match attr.meta_kind() { + Some(MetaItemKind::NameValue(lit)) => { + InlineAttr::Force { attr_span: attr.span, reason: Some(lit.symbol) } + } + Some(MetaItemKind::Word) => { + InlineAttr::Force { attr_span: attr.span, reason: None } + } + _ => { + debug!("`rustc_force_inline` not checked by attribute validation"); + ia + } + } + } else { + ia } }); @@ -601,7 +626,7 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { // is probably a poor usage of `#[inline(always)]` and easily avoided by not using the attribute. if tcx.features().target_feature_11() && tcx.is_closure_like(did.to_def_id()) - && codegen_fn_attrs.inline != InlineAttr::Always + && !codegen_fn_attrs.inline.always() { let owner_id = tcx.parent(did.to_def_id()); if tcx.def_kind(owner_id).has_codegen_attrs() { @@ -611,11 +636,15 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { } } - // If a function uses #[target_feature] it can't be inlined into general + // If a function uses `#[target_feature]` it can't be inlined into general // purpose functions as they wouldn't have the right target features - // enabled. For that reason we also forbid #[inline(always)] as it can't be + // enabled. For that reason we also forbid `#[inline(always)]` as it can't be // respected. - if !codegen_fn_attrs.target_features.is_empty() && codegen_fn_attrs.inline == InlineAttr::Always + // + // `#[rustc_force_inline]` doesn't need to be prohibited here, that + // is implemented entirely in rustc can attempt to inline and error if it cannot. + if !codegen_fn_attrs.target_features.is_empty() + && matches!(codegen_fn_attrs.inline, InlineAttr::Always) { if let Some(span) = inline_span { tcx.dcx().span_err( @@ -626,7 +655,7 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { } } - if !codegen_fn_attrs.no_sanitize.is_empty() && codegen_fn_attrs.inline == InlineAttr::Always { + if !codegen_fn_attrs.no_sanitize.is_empty() && codegen_fn_attrs.inline.always() { if let (Some(no_sanitize_span), Some(inline_span)) = (no_sanitize_span, inline_span) { let hir_id = tcx.local_def_id_to_hir_id(did); tcx.node_span_lint( diff --git a/compiler/rustc_feature/src/builtin_attrs.rs b/compiler/rustc_feature/src/builtin_attrs.rs index 3bf485c2eb6fb..33c1dc41d0838 100644 --- a/compiler/rustc_feature/src/builtin_attrs.rs +++ b/compiler/rustc_feature/src/builtin_attrs.rs @@ -1014,6 +1014,10 @@ pub const BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[ rustc_no_mir_inline, Normal, template!(Word), WarnFollowing, EncodeCrossCrate::Yes, "#[rustc_no_mir_inline] prevents the MIR inliner from inlining a function while not affecting codegen" ), + rustc_attr!( + rustc_force_inline, Normal, template!(Word, NameValueStr: "reason"), WarnFollowing, EncodeCrossCrate::Yes, + "#![rustc_force_inline] forces a free function to be inlined" + ), // ========================================================================== // Internal attributes, Testing: diff --git a/compiler/rustc_hir_typeck/src/coercion.rs b/compiler/rustc_hir_typeck/src/coercion.rs index bf41dcbe4a365..4044bf7778dfb 100644 --- a/compiler/rustc_hir_typeck/src/coercion.rs +++ b/compiler/rustc_hir_typeck/src/coercion.rs @@ -38,6 +38,7 @@ use std::ops::Deref; use rustc_abi::ExternAbi; +use rustc_attr::InlineAttr; use rustc_errors::codes::*; use rustc_errors::{Applicability, Diag, struct_span_code_err}; use rustc_hir as hir; @@ -923,11 +924,14 @@ impl<'f, 'tcx> Coerce<'f, 'tcx> { return Err(TypeError::IntrinsicCast); } - // Safe `#[target_feature]` functions are not assignable to safe fn pointers (RFC 2396). + let fn_attrs = self.tcx.codegen_fn_attrs(def_id); + if matches!(fn_attrs.inline, rustc_attr::InlineAttr::Force { .. }) { + return Err(TypeError::ForceInlineCast); + } - if b_hdr.safety == hir::Safety::Safe - && !self.tcx.codegen_fn_attrs(def_id).target_features.is_empty() - { + // Safe `#[target_feature]` functions are not assignable to safe fn pointers + // (RFC 2396). + if b_hdr.safety == hir::Safety::Safe && !fn_attrs.target_features.is_empty() { return Err(TypeError::TargetFeatureCast(def_id)); } } @@ -1194,6 +1198,17 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { return Ok(prev_ty); } + let is_force_inline = |ty: Ty<'tcx>| { + if let ty::FnDef(did, _) = ty.kind() { + matches!(self.tcx.codegen_fn_attrs(did).inline, InlineAttr::Force { .. }) + } else { + false + } + }; + if is_force_inline(prev_ty) || is_force_inline(new_ty) { + return Err(TypeError::ForceInlineCast); + } + // Special-case that coercion alone cannot handle: // Function items or non-capturing closures of differing IDs or GenericArgs. let (a_sig, b_sig) = { diff --git a/compiler/rustc_lint_defs/src/builtin.rs b/compiler/rustc_lint_defs/src/builtin.rs index 54e927df3c42b..e917a5d16d20b 100644 --- a/compiler/rustc_lint_defs/src/builtin.rs +++ b/compiler/rustc_lint_defs/src/builtin.rs @@ -46,6 +46,7 @@ declare_lint_pass! { EXPORTED_PRIVATE_DEPENDENCIES, FFI_UNWIND_CALLS, FORBIDDEN_LINT_GROUPS, + FORCED_INLINE, FUNCTION_ITEM_REFERENCES, FUZZY_PROVENANCE_CASTS, HIDDEN_GLOB_REEXPORTS, @@ -5178,3 +5179,31 @@ declare_lint! { reference: "issue #116558 ", }; } + +declare_lint! { + /// The `forced_inline` lint is emitted when a function annotated with + /// `#[rustc_force_inline]` was not able to be inlined. + /// + /// ### Example + /// + /// ```rust,ignore (needs rustc_attrs) + /// #[rustc_no_mir_inline] + /// #[rustc_force_inline] + /// fn foo() { } + /// + /// fn bar() { foo() } + /// ``` + /// + /// ### Explanation + /// + /// Functions can be marked as `#[rustc_force_inline]` in the standard + /// library if they are required to be inlined in order to uphold + /// security properties or some other similar guarantee. + /// + /// In some circumstances, these functions cannot be inlined and a + /// reason will be provided, this can either be rectified or the + /// lint can be silenced if the risk is acceptable. + pub FORCED_INLINE, + Deny, + "`#[rustc_force_inline]`-annotated function could not be inlined" +} diff --git a/compiler/rustc_middle/src/mir/mono.rs b/compiler/rustc_middle/src/mir/mono.rs index 1f50b67cb50d5..862df78a69f63 100644 --- a/compiler/rustc_middle/src/mir/mono.rs +++ b/compiler/rustc_middle/src/mir/mono.rs @@ -1,7 +1,6 @@ use std::fmt; use std::hash::Hash; -use rustc_attr::InlineAttr; use rustc_data_structures::base_n::{BaseNString, CASE_INSENSITIVE, ToBaseN}; use rustc_data_structures::fingerprint::Fingerprint; use rustc_data_structures::fx::FxIndexMap; @@ -114,7 +113,8 @@ impl<'tcx> MonoItem<'tcx> { return InstantiationMode::GloballyShared { may_conflict: false }; } - if let InlineAttr::Never = tcx.codegen_fn_attrs(instance.def_id()).inline + if let rustc_attr::InlineAttr::Never = + tcx.codegen_fn_attrs(instance.def_id()).inline && self.is_generic_fn() { // Upgrade inline(never) to a globally shared instance. @@ -133,9 +133,10 @@ impl<'tcx> MonoItem<'tcx> { // creating one copy of this `#[inline]` function which may // conflict with upstream crates as it could be an exported // symbol. - match tcx.codegen_fn_attrs(instance.def_id()).inline { - InlineAttr::Always => InstantiationMode::LocalCopy, - _ => InstantiationMode::GloballyShared { may_conflict: true }, + if tcx.codegen_fn_attrs(instance.def_id()).inline.always() { + InstantiationMode::LocalCopy + } else { + InstantiationMode::GloballyShared { may_conflict: true } } } MonoItem::Static(..) | MonoItem::GlobalAsm(..) => { diff --git a/compiler/rustc_middle/src/ty/error.rs b/compiler/rustc_middle/src/ty/error.rs index 4a82af3255950..2c32ada1ec1cb 100644 --- a/compiler/rustc_middle/src/ty/error.rs +++ b/compiler/rustc_middle/src/ty/error.rs @@ -109,6 +109,9 @@ impl<'tcx> TypeError<'tcx> { TypeError::ConstMismatch(ref values) => { format!("expected `{}`, found `{}`", values.expected, values.found).into() } + TypeError::ForceInlineCast => { + "cannot coerce functions which must be inlined to function pointers".into() + } TypeError::IntrinsicCast => "cannot coerce intrinsics to function pointers".into(), TypeError::TargetFeatureCast(_) => { "cannot coerce functions with `#[target_feature]` to safe function pointers".into() diff --git a/compiler/rustc_mir_transform/messages.ftl b/compiler/rustc_mir_transform/messages.ftl index d00bfc66a6a57..b0c023cca8242 100644 --- a/compiler/rustc_mir_transform/messages.ftl +++ b/compiler/rustc_mir_transform/messages.ftl @@ -19,6 +19,17 @@ mir_transform_ffi_unwind_call = call to {$foreign -> mir_transform_fn_item_ref = taking a reference to a function item does not give a function pointer .suggestion = cast `{$ident}` to obtain a function pointer +mir_transform_force_inline = + `{$callee}` could not be inlined into `{$caller}` but is required to be inlined + .call = ...`{$callee}` called here + .attr = inlining due to this annotation + .caller = within `{$caller}`... + .callee = `{$callee}` defined here + .note = could not be inlined due to: {$reason} + +mir_transform_force_inline_justification = + `{$callee}` is required to be inlined to: {$sym} + mir_transform_must_not_suspend = {$pre}`{$def_path}`{$post} held across a suspend point, but should not be .label = the value is held across this suspend point .note = {$reason} diff --git a/compiler/rustc_mir_transform/src/cross_crate_inline.rs b/compiler/rustc_mir_transform/src/cross_crate_inline.rs index 589be81558c2b..d0ec106cfc721 100644 --- a/compiler/rustc_mir_transform/src/cross_crate_inline.rs +++ b/compiler/rustc_mir_transform/src/cross_crate_inline.rs @@ -46,7 +46,7 @@ fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool { // #[inline(never)] to force code generation. match codegen_fn_attrs.inline { InlineAttr::Never => return false, - InlineAttr::Hint | InlineAttr::Always => return true, + InlineAttr::Hint | InlineAttr::Always | InlineAttr::Force { .. } => return true, _ => {} } @@ -69,8 +69,9 @@ fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool { // Don't do any inference if codegen optimizations are disabled and also MIR inlining is not // enabled. This ensures that we do inference even if someone only passes -Zinline-mir, // which is less confusing than having to also enable -Copt-level=1. - if matches!(tcx.sess.opts.optimize, OptLevel::No) && !pm::should_run_pass(tcx, &inline::Inline) - { + let inliner_will_run = pm::should_run_pass(tcx, &inline::Inline) + || inline::ForceInline::should_run_pass_for_callee(tcx, def_id.to_def_id()); + if matches!(tcx.sess.opts.optimize, OptLevel::No) && !inliner_will_run { return false; } diff --git a/compiler/rustc_mir_transform/src/errors.rs b/compiler/rustc_mir_transform/src/errors.rs index 2d9eeddea2e22..015633d145f15 100644 --- a/compiler/rustc_mir_transform/src/errors.rs +++ b/compiler/rustc_mir_transform/src/errors.rs @@ -4,8 +4,8 @@ use rustc_macros::{Diagnostic, LintDiagnostic, Subdiagnostic}; use rustc_middle::mir::AssertKind; use rustc_middle::ty::TyCtxt; use rustc_session::lint::{self, Lint}; -use rustc_span::Span; use rustc_span::def_id::DefId; +use rustc_span::{Span, Symbol}; use crate::fluent_generated as fluent; @@ -142,3 +142,29 @@ pub(crate) struct MustNotSuspendReason { #[note(mir_transform_note2)] #[help] pub(crate) struct UndefinedTransmute; + +#[derive(Diagnostic)] +#[diag(mir_transform_force_inline)] +#[note] +pub(crate) struct ForceInlineFailure { + #[label(mir_transform_caller)] + pub caller_span: Span, + #[label(mir_transform_callee)] + pub callee_span: Span, + #[label(mir_transform_attr)] + pub attr_span: Span, + #[primary_span] + #[label(mir_transform_call)] + pub call_span: Span, + pub callee: String, + pub caller: String, + pub reason: &'static str, + #[subdiagnostic] + pub justification: Option, +} + +#[derive(Subdiagnostic)] +#[note(mir_transform_force_inline_justification)] +pub(crate) struct ForceInlineJustification { + pub sym: Symbol, +} diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs index 79c38b50459c4..b818fe630c1d9 100644 --- a/compiler/rustc_mir_transform/src/inline.rs +++ b/compiler/rustc_mir_transform/src/inline.rs @@ -29,10 +29,6 @@ pub(crate) mod cycle; const TOP_DOWN_DEPTH_LIMIT: usize = 5; -// Made public so that `mir_drops_elaborated_and_const_checked` can be overridden -// by custom rustc drivers, running all the steps by themselves. See #114628. -pub struct Inline; - #[derive(Clone, Debug)] struct CallSite<'tcx> { callee: Instance<'tcx>, @@ -41,14 +37,12 @@ struct CallSite<'tcx> { source_info: SourceInfo, } +// Made public so that `mir_drops_elaborated_and_const_checked` can be overridden +// by custom rustc drivers, running all the steps by themselves. See #114628. +pub struct Inline; + impl<'tcx> crate::MirPass<'tcx> for Inline { fn is_enabled(&self, sess: &rustc_session::Session) -> bool { - // FIXME(#127234): Coverage instrumentation currently doesn't handle inlined - // MIR correctly when Modified Condition/Decision Coverage is enabled. - if sess.instrument_coverage_mcdc() { - return false; - } - if let Some(enabled) = sess.opts.unstable_opts.inline_mir { return enabled; } @@ -67,7 +61,7 @@ impl<'tcx> crate::MirPass<'tcx> for Inline { fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { let span = trace_span!("inline", body = %tcx.def_path_str(body.source.def_id())); let _guard = span.enter(); - if inline(tcx, body) { + if inline::>(tcx, body) { debug!("running simplify cfg on {:?}", body.source); simplify_cfg(body); deref_finder(tcx, body); @@ -75,47 +69,84 @@ impl<'tcx> crate::MirPass<'tcx> for Inline { } } -fn inline<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> bool { - let def_id = body.source.def_id().expect_local(); +pub struct ForceInline; - // Only do inlining into fn bodies. - if !tcx.hir().body_owner_kind(def_id).is_fn_or_closure() { - return false; +impl ForceInline { + pub fn should_run_pass_for_callee<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> bool { + matches!(tcx.codegen_fn_attrs(def_id).inline, InlineAttr::Force { .. }) } - if body.source.promoted.is_some() { - return false; +} + +impl<'tcx> crate::MirPass<'tcx> for ForceInline { + fn is_enabled(&self, _: &rustc_session::Session) -> bool { + true } - // Avoid inlining into coroutines, since their `optimized_mir` is used for layout computation, - // which can create a cycle, even when no attempt is made to inline the function in the other - // direction. - if body.coroutine.is_some() { - return false; + + fn can_be_overridden(&self) -> bool { + false } - let typing_env = body.typing_env(tcx); - let codegen_fn_attrs = tcx.codegen_fn_attrs(def_id); + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + let span = trace_span!("force_inline", body = %tcx.def_path_str(body.source.def_id())); + let _guard = span.enter(); + if inline::>(tcx, body) { + debug!("running simplify cfg on {:?}", body.source); + simplify_cfg(body); + deref_finder(tcx, body); + } + } +} - let mut this = Inliner { - tcx, - typing_env, - codegen_fn_attrs, - history: Vec::new(), - changed: false, - caller_is_inline_forwarder: matches!( - codegen_fn_attrs.inline, - InlineAttr::Hint | InlineAttr::Always - ) && body_is_forwarder(body), - }; - let blocks = START_BLOCK..body.basic_blocks.next_index(); - this.process_blocks(body, blocks); - this.changed +trait Inliner<'tcx> { + fn new(tcx: TyCtxt<'tcx>, def_id: DefId, body: &Body<'tcx>) -> Self; + + fn tcx(&self) -> TyCtxt<'tcx>; + fn typing_env(&self) -> ty::TypingEnv<'tcx>; + fn history(&self) -> &[DefId]; + fn caller_def_id(&self) -> DefId; + + /// Has the caller body been changed? + fn changed(self) -> bool; + + /// Should inlining happen for a given callee? + fn should_inline_for_callee(&self, def_id: DefId) -> bool; + + fn check_caller_mir_body(&self, body: &Body<'tcx>) -> bool; + + /// Returns inlining decision that is based on the examination of callee MIR body. + /// Assumes that codegen attributes have been checked for compatibility already. + fn check_callee_mir_body( + &self, + callsite: &CallSite<'tcx>, + callee_body: &Body<'tcx>, + callee_attrs: &CodegenFnAttrs, + cross_crate_inlinable: bool, + ) -> Result<(), &'static str>; + + // How many callsites in a body are we allowed to inline? We need to limit this in order + // to prevent super-linear growth in MIR size. + fn inline_limit_for_block(&self) -> Option; + + /// Called when inlining succeeds. + fn on_inline_success( + &mut self, + callsite: &CallSite<'tcx>, + caller_body: &mut Body<'tcx>, + new_blocks: std::ops::Range, + ); + + /// Called when inlining failed or was not performed. + fn on_inline_failure(&self, callsite: &CallSite<'tcx>, reason: &'static str); + + /// Called when the inline limit for a body is reached. + fn on_inline_limit_reached(&self) -> bool; } -struct Inliner<'tcx> { +struct ForceInliner<'tcx> { tcx: TyCtxt<'tcx>, typing_env: ty::TypingEnv<'tcx>, - /// Caller codegen attributes. - codegen_fn_attrs: &'tcx CodegenFnAttrs, + /// `DefId` of caller. + def_id: DefId, /// Stack of inlined instances. /// We only check the `DefId` and not the args because we want to /// avoid inlining cases of polymorphic recursion. @@ -124,365 +155,213 @@ struct Inliner<'tcx> { history: Vec, /// Indicates that the caller body has been modified. changed: bool, - /// Indicates that the caller is #[inline] and just calls another function, - /// and thus we can inline less into it as it'll be inlined itself. - caller_is_inline_forwarder: bool, } -impl<'tcx> Inliner<'tcx> { - fn process_blocks(&mut self, caller_body: &mut Body<'tcx>, blocks: Range) { - // How many callsites in this body are we allowed to inline? We need to limit this in order - // to prevent super-linear growth in MIR size - let inline_limit = match self.history.len() { - 0 => usize::MAX, - 1..=TOP_DOWN_DEPTH_LIMIT => 1, - _ => return, - }; - let mut inlined_count = 0; - for bb in blocks { - let bb_data = &caller_body[bb]; - if bb_data.is_cleanup { - continue; - } - - let Some(callsite) = self.resolve_callsite(caller_body, bb, bb_data) else { - continue; - }; - - let span = trace_span!("process_blocks", %callsite.callee, ?bb); - let _guard = span.enter(); - - match self.try_inlining(caller_body, &callsite) { - Err(reason) => { - debug!("not-inlined {} [{}]", callsite.callee, reason); - } - Ok(new_blocks) => { - debug!("inlined {}", callsite.callee); - self.changed = true; - - self.history.push(callsite.callee.def_id()); - self.process_blocks(caller_body, new_blocks); - self.history.pop(); - - inlined_count += 1; - if inlined_count == inline_limit { - debug!("inline count reached"); - return; - } - } - } - } +impl<'tcx> Inliner<'tcx> for ForceInliner<'tcx> { + fn new(tcx: TyCtxt<'tcx>, def_id: DefId, body: &Body<'tcx>) -> Self { + Self { tcx, typing_env: body.typing_env(tcx), def_id, history: Vec::new(), changed: false } } - /// Attempts to inline a callsite into the caller body. When successful returns basic blocks - /// containing the inlined body. Otherwise returns an error describing why inlining didn't take - /// place. - fn try_inlining( - &self, - caller_body: &mut Body<'tcx>, - callsite: &CallSite<'tcx>, - ) -> Result, &'static str> { - self.check_mir_is_available(caller_body, callsite.callee)?; - - let callee_attrs = self.tcx.codegen_fn_attrs(callsite.callee.def_id()); - let cross_crate_inlinable = self.tcx.cross_crate_inlinable(callsite.callee.def_id()); - self.check_codegen_attributes(callsite, callee_attrs, cross_crate_inlinable)?; - - // Intrinsic fallback bodies are automatically made cross-crate inlineable, - // but at this stage we don't know whether codegen knows the intrinsic, - // so just conservatively don't inline it. - if self.tcx.has_attr(callsite.callee.def_id(), sym::rustc_intrinsic) { - return Err("Callee is an intrinsic, do not inline fallback bodies"); - } - - let terminator = caller_body[callsite.block].terminator.as_ref().unwrap(); - let TerminatorKind::Call { args, destination, .. } = &terminator.kind else { bug!() }; - let destination_ty = destination.ty(&caller_body.local_decls, self.tcx).ty; - for arg in args { - if !arg.node.ty(&caller_body.local_decls, self.tcx).is_sized(self.tcx, self.typing_env) - { - // We do not allow inlining functions with unsized params. Inlining these functions - // could create unsized locals, which are unsound and being phased out. - return Err("Call has unsized argument"); - } - } - - let callee_body = try_instance_mir(self.tcx, callsite.callee.def)?; - self.check_mir_body(callsite, callee_body, callee_attrs, cross_crate_inlinable)?; - - let Ok(callee_body) = callsite.callee.try_instantiate_mir_and_normalize_erasing_regions( - self.tcx, - self.typing_env, - ty::EarlyBinder::bind(callee_body.clone()), - ) else { - return Err("failed to normalize callee body"); - }; - - // Normally, this shouldn't be required, but trait normalization failure can create a - // validation ICE. - if !validate_types(self.tcx, self.typing_env, &callee_body, &caller_body).is_empty() { - return Err("failed to validate callee body"); - } - - // Check call signature compatibility. - // Normally, this shouldn't be required, but trait normalization failure can create a - // validation ICE. - let output_type = callee_body.return_ty(); - if !util::sub_types(self.tcx, self.typing_env, output_type, destination_ty) { - trace!(?output_type, ?destination_ty); - return Err("failed to normalize return type"); - } - if callsite.fn_sig.abi() == ExternAbi::RustCall { - // FIXME: Don't inline user-written `extern "rust-call"` functions, - // since this is generally perf-negative on rustc, and we hope that - // LLVM will inline these functions instead. - if callee_body.spread_arg.is_some() { - return Err("do not inline user-written rust-call functions"); - } + fn tcx(&self) -> TyCtxt<'tcx> { + self.tcx + } - let (self_arg, arg_tuple) = match &args[..] { - [arg_tuple] => (None, arg_tuple), - [self_arg, arg_tuple] => (Some(self_arg), arg_tuple), - _ => bug!("Expected `rust-call` to have 1 or 2 args"), - }; + fn typing_env(&self) -> ty::TypingEnv<'tcx> { + self.typing_env + } - let self_arg_ty = - self_arg.map(|self_arg| self_arg.node.ty(&caller_body.local_decls, self.tcx)); + fn history(&self) -> &[DefId] { + &self.history + } - let arg_tuple_ty = arg_tuple.node.ty(&caller_body.local_decls, self.tcx); - let ty::Tuple(arg_tuple_tys) = *arg_tuple_ty.kind() else { - bug!("Closure arguments are not passed as a tuple"); - }; + fn caller_def_id(&self) -> DefId { + self.def_id + } - for (arg_ty, input) in - self_arg_ty.into_iter().chain(arg_tuple_tys).zip(callee_body.args_iter()) - { - let input_type = callee_body.local_decls[input].ty; - if !util::sub_types(self.tcx, self.typing_env, input_type, arg_ty) { - trace!(?arg_ty, ?input_type); - return Err("failed to normalize tuple argument type"); - } - } - } else { - for (arg, input) in args.iter().zip(callee_body.args_iter()) { - let input_type = callee_body.local_decls[input].ty; - let arg_ty = arg.node.ty(&caller_body.local_decls, self.tcx); - if !util::sub_types(self.tcx, self.typing_env, input_type, arg_ty) { - trace!(?arg_ty, ?input_type); - return Err("failed to normalize argument type"); - } - } - } + fn changed(self) -> bool { + self.changed + } - let old_blocks = caller_body.basic_blocks.next_index(); - self.inline_call(caller_body, callsite, callee_body); - let new_blocks = old_blocks..caller_body.basic_blocks.next_index(); + fn should_inline_for_callee(&self, def_id: DefId) -> bool { + ForceInline::should_run_pass_for_callee(self.tcx(), def_id) + } - Ok(new_blocks) + fn check_caller_mir_body(&self, _: &Body<'tcx>) -> bool { + true } - fn check_mir_is_available( + #[instrument(level = "debug", skip(self, callee_body))] + fn check_callee_mir_body( &self, - caller_body: &Body<'tcx>, - callee: Instance<'tcx>, + _: &CallSite<'tcx>, + callee_body: &Body<'tcx>, + callee_attrs: &CodegenFnAttrs, + _: bool, ) -> Result<(), &'static str> { - let caller_def_id = caller_body.source.def_id(); - let callee_def_id = callee.def_id(); - if callee_def_id == caller_def_id { - return Err("self-recursion"); - } - - match callee.def { - InstanceKind::Item(_) => { - // If there is no MIR available (either because it was not in metadata or - // because it has no MIR because it's an extern function), then the inliner - // won't cause cycles on this. - if !self.tcx.is_mir_available(callee_def_id) { - return Err("item MIR unavailable"); - } - } - // These have no own callable MIR. - InstanceKind::Intrinsic(_) | InstanceKind::Virtual(..) => { - return Err("instance without MIR (intrinsic / virtual)"); - } - - // FIXME(#127030): `ConstParamHasTy` has bad interactions with - // the drop shim builder, which does not evaluate predicates in - // the correct param-env for types being dropped. Stall resolving - // the MIR for this instance until all of its const params are - // substituted. - InstanceKind::DropGlue(_, Some(ty)) if ty.has_type_flags(TypeFlags::HAS_CT_PARAM) => { - return Err("still needs substitution"); - } - - // This cannot result in an immediate cycle since the callee MIR is a shim, which does - // not get any optimizations run on it. Any subsequent inlining may cause cycles, but we - // do not need to catch this here, we can wait until the inliner decides to continue - // inlining a second time. - InstanceKind::VTableShim(_) - | InstanceKind::ReifyShim(..) - | InstanceKind::FnPtrShim(..) - | InstanceKind::ClosureOnceShim { .. } - | InstanceKind::ConstructCoroutineInClosureShim { .. } - | InstanceKind::DropGlue(..) - | InstanceKind::CloneShim(..) - | InstanceKind::ThreadLocalShim(..) - | InstanceKind::FnPtrAddrShim(..) - | InstanceKind::AsyncDropGlueCtorShim(..) => return Ok(()), + if callee_body.tainted_by_errors.is_some() { + return Err("body has errors"); } - if self.tcx.is_constructor(callee_def_id) { - trace!("constructors always have MIR"); - // Constructor functions cannot cause a query cycle. - return Ok(()); - } - - if callee_def_id.is_local() { - // If we know for sure that the function we're calling will itself try to - // call us, then we avoid inlining that function. - if self.tcx.mir_callgraph_reachable((callee, caller_def_id.expect_local())) { - return Err("caller might be reachable from callee (query cycle avoidance)"); - } - - Ok(()) + let caller_attrs = self.tcx().codegen_fn_attrs(self.caller_def_id()); + if callee_attrs.instruction_set != caller_attrs.instruction_set + && callee_body + .basic_blocks + .iter() + .any(|bb| matches!(bb.terminator().kind, TerminatorKind::InlineAsm { .. })) + { + // During the attribute checking stage we allow a callee with no + // instruction_set assigned to count as compatible with a function that does + // assign one. However, during this stage we require an exact match when any + // inline-asm is detected. LLVM will still possibly do an inline later on + // if the no-attribute function ends up with the same instruction set anyway. + Err("cannot move inline-asm across instruction sets") + } else if callee_body + .basic_blocks + .iter() + .any(|bb| matches!(bb.terminator().kind, TerminatorKind::TailCall { .. })) + { + // FIXME(explicit_tail_calls): figure out how exactly functions containing tail + // calls can be inlined (and if they even should) + Err("can't inline functions with tail calls") } else { - // This cannot result in an immediate cycle since the callee MIR is from another crate - // and is already optimized. Any subsequent inlining may cause cycles, but we do - // not need to catch this here, we can wait until the inliner decides to continue - // inlining a second time. - trace!("functions from other crates always have MIR"); Ok(()) } } - fn resolve_callsite( - &self, - caller_body: &Body<'tcx>, - bb: BasicBlock, - bb_data: &BasicBlockData<'tcx>, - ) -> Option> { - // Only consider direct calls to functions - let terminator = bb_data.terminator(); - - // FIXME(explicit_tail_calls): figure out if we can inline tail calls - if let TerminatorKind::Call { ref func, fn_span, .. } = terminator.kind { - let func_ty = func.ty(caller_body, self.tcx); - if let ty::FnDef(def_id, args) = *func_ty.kind() { - // To resolve an instance its args have to be fully normalized. - let args = self.tcx.try_normalize_erasing_regions(self.typing_env, args).ok()?; - let callee = Instance::try_resolve(self.tcx, self.typing_env, def_id, args) - .ok() - .flatten()?; - - if let InstanceKind::Virtual(..) | InstanceKind::Intrinsic(_) = callee.def { - return None; - } - - if self.history.contains(&callee.def_id()) { - return None; - } + fn inline_limit_for_block(&self) -> Option { + Some(usize::MAX) + } - let fn_sig = self.tcx.fn_sig(def_id).instantiate(self.tcx, args); + fn on_inline_success( + &mut self, + callsite: &CallSite<'tcx>, + caller_body: &mut Body<'tcx>, + new_blocks: std::ops::Range, + ) { + self.changed = true; - // Additionally, check that the body that we're inlining actually agrees - // with the ABI of the trait that the item comes from. - if let InstanceKind::Item(instance_def_id) = callee.def - && self.tcx.def_kind(instance_def_id) == DefKind::AssocFn - && let instance_fn_sig = self.tcx.fn_sig(instance_def_id).skip_binder() - && instance_fn_sig.abi() != fn_sig.abi() - { - return None; - } + self.history.push(callsite.callee.def_id()); + process_blocks(self, caller_body, new_blocks); + self.history.pop(); + } - let source_info = SourceInfo { span: fn_span, ..terminator.source_info }; + fn on_inline_failure(&self, callsite: &CallSite<'tcx>, reason: &'static str) { + let tcx = self.tcx(); + let InlineAttr::Force { attr_span, reason: justification } = + tcx.codegen_fn_attrs(callsite.callee.def_id()).inline + else { + bug!("called on item without required inlining"); + }; - return Some(CallSite { callee, fn_sig, block: bb, source_info }); - } - } + let call_span = callsite.source_info.span; + tcx.dcx().emit_err(crate::errors::ForceInlineFailure { + call_span, + attr_span, + caller_span: tcx.def_span(self.def_id), + caller: tcx.def_path_str(self.def_id), + callee_span: tcx.def_span(callsite.callee.def_id()), + callee: tcx.def_path_str(callsite.callee.def_id()), + reason, + justification: justification.map(|sym| crate::errors::ForceInlineJustification { sym }), + }); + } - None + fn on_inline_limit_reached(&self) -> bool { + false } +} - /// Returns an error if inlining is not possible based on codegen attributes alone. A success - /// indicates that inlining decision should be based on other criteria. - fn check_codegen_attributes( - &self, - callsite: &CallSite<'tcx>, - callee_attrs: &CodegenFnAttrs, - cross_crate_inlinable: bool, - ) -> Result<(), &'static str> { - if self.tcx.has_attr(callsite.callee.def_id(), sym::rustc_no_mir_inline) { - return Err("#[rustc_no_mir_inline]"); - } +struct NormalInliner<'tcx> { + tcx: TyCtxt<'tcx>, + typing_env: ty::TypingEnv<'tcx>, + /// `DefId` of caller. + def_id: DefId, + /// Stack of inlined instances. + /// We only check the `DefId` and not the args because we want to + /// avoid inlining cases of polymorphic recursion. + /// The number of `DefId`s is finite, so checking history is enough + /// to ensure that we do not loop endlessly while inlining. + history: Vec, + /// Indicates that the caller body has been modified. + changed: bool, + /// Indicates that the caller is #[inline] and just calls another function, + /// and thus we can inline less into it as it'll be inlined itself. + caller_is_inline_forwarder: bool, +} - if let InlineAttr::Never = callee_attrs.inline { - return Err("never inline hint"); +impl<'tcx> Inliner<'tcx> for NormalInliner<'tcx> { + fn new(tcx: TyCtxt<'tcx>, def_id: DefId, body: &Body<'tcx>) -> Self { + let typing_env = body.typing_env(tcx); + let codegen_fn_attrs = tcx.codegen_fn_attrs(def_id); + + Self { + tcx, + typing_env, + def_id, + history: Vec::new(), + changed: false, + caller_is_inline_forwarder: matches!( + codegen_fn_attrs.inline, + InlineAttr::Hint | InlineAttr::Always | InlineAttr::Force { .. } + ) && body_is_forwarder(body), } + } - // Reachability pass defines which functions are eligible for inlining. Generally inlining - // other functions is incorrect because they could reference symbols that aren't exported. - let is_generic = callsite.callee.args.non_erasable_generics().next().is_some(); - if !is_generic && !cross_crate_inlinable { - return Err("not exported"); - } + fn tcx(&self) -> TyCtxt<'tcx> { + self.tcx + } - if callsite.fn_sig.c_variadic() { - return Err("C variadic"); - } + fn caller_def_id(&self) -> DefId { + self.def_id + } - if callee_attrs.flags.contains(CodegenFnAttrFlags::COLD) { - return Err("cold"); - } + fn typing_env(&self) -> ty::TypingEnv<'tcx> { + self.typing_env + } - if callee_attrs.no_sanitize != self.codegen_fn_attrs.no_sanitize { - return Err("incompatible sanitizer set"); - } + fn history(&self) -> &[DefId] { + &self.history + } - // Two functions are compatible if the callee has no attribute (meaning - // that it's codegen agnostic), or sets an attribute that is identical - // to this function's attribute. - if callee_attrs.instruction_set.is_some() - && callee_attrs.instruction_set != self.codegen_fn_attrs.instruction_set - { - return Err("incompatible instruction set"); - } + fn changed(self) -> bool { + self.changed + } - let callee_feature_names = callee_attrs.target_features.iter().map(|f| f.name); - let this_feature_names = self.codegen_fn_attrs.target_features.iter().map(|f| f.name); - if callee_feature_names.ne(this_feature_names) { - // In general it is not correct to inline a callee with target features that are a - // subset of the caller. This is because the callee might contain calls, and the ABI of - // those calls depends on the target features of the surrounding function. By moving a - // `Call` terminator from one MIR body to another with more target features, we might - // change the ABI of that call! - return Err("incompatible target features"); + fn should_inline_for_callee(&self, _: DefId) -> bool { + true + } + + fn check_caller_mir_body(&self, body: &Body<'tcx>) -> bool { + // Avoid inlining into coroutines, since their `optimized_mir` is used for layout computation, + // which can create a cycle, even when no attempt is made to inline the function in the other + // direction. + if body.coroutine.is_some() { + return false; } - Ok(()) + true } - /// Returns inlining decision that is based on the examination of callee MIR body. - /// Assumes that codegen attributes have been checked for compatibility already. #[instrument(level = "debug", skip(self, callee_body))] - fn check_mir_body( + fn check_callee_mir_body( &self, callsite: &CallSite<'tcx>, callee_body: &Body<'tcx>, callee_attrs: &CodegenFnAttrs, cross_crate_inlinable: bool, ) -> Result<(), &'static str> { - let tcx = self.tcx; + let tcx = self.tcx(); if let Some(_) = callee_body.tainted_by_errors { - return Err("Body is tainted"); + return Err("body has errors"); } let mut threshold = if self.caller_is_inline_forwarder { - self.tcx.sess.opts.unstable_opts.inline_mir_forwarder_threshold.unwrap_or(30) + tcx.sess.opts.unstable_opts.inline_mir_forwarder_threshold.unwrap_or(30) } else if cross_crate_inlinable { - self.tcx.sess.opts.unstable_opts.inline_mir_hint_threshold.unwrap_or(100) + tcx.sess.opts.unstable_opts.inline_mir_hint_threshold.unwrap_or(100) } else { - self.tcx.sess.opts.unstable_opts.inline_mir_threshold.unwrap_or(50) + tcx.sess.opts.unstable_opts.inline_mir_threshold.unwrap_or(50) }; // Give a bonus functions with a small number of blocks, @@ -496,7 +375,7 @@ impl<'tcx> Inliner<'tcx> { // FIXME: Give a bonus to functions with only a single caller let mut checker = - CostChecker::new(self.tcx, self.typing_env, Some(callsite.callee), callee_body); + CostChecker::new(tcx, self.typing_env(), Some(callsite.callee), callee_body); checker.add_function_level_costs(); @@ -512,20 +391,20 @@ impl<'tcx> Inliner<'tcx> { checker.visit_basic_block_data(bb, blk); let term = blk.terminator(); + let caller_attrs = tcx.codegen_fn_attrs(self.caller_def_id()); if let TerminatorKind::Drop { ref place, target, unwind, replace: _ } = term.kind { work_list.push(target); // If the place doesn't actually need dropping, treat it like a regular goto. - let ty = callsite.callee.instantiate_mir( - self.tcx, - ty::EarlyBinder::bind(&place.ty(callee_body, tcx).ty), - ); - if ty.needs_drop(tcx, self.typing_env) + let ty = callsite + .callee + .instantiate_mir(tcx, ty::EarlyBinder::bind(&place.ty(callee_body, tcx).ty)); + if ty.needs_drop(tcx, self.typing_env()) && let UnwindAction::Cleanup(unwind) = unwind { work_list.push(unwind); } - } else if callee_attrs.instruction_set != self.codegen_fn_attrs.instruction_set + } else if callee_attrs.instruction_set != caller_attrs.instruction_set && matches!(term.kind, TerminatorKind::InlineAsm { .. }) { // During the attribute checking stage we allow a callee with no @@ -533,7 +412,7 @@ impl<'tcx> Inliner<'tcx> { // assign one. However, during this stage we require an exact match when any // inline-asm is detected. LLVM will still possibly do an inline later on // if the no-attribute function ends up with the same instruction set anyway. - return Err("Cannot move inline-asm across instruction sets"); + return Err("cannot move inline-asm across instruction sets"); } else if let TerminatorKind::TailCall { .. } = term.kind { // FIXME(explicit_tail_calls): figure out how exactly functions containing tail // calls can be inlined (and if they even should) @@ -557,319 +436,711 @@ impl<'tcx> Inliner<'tcx> { } } - fn inline_call( - &self, - caller_body: &mut Body<'tcx>, + fn inline_limit_for_block(&self) -> Option { + match self.history.len() { + 0 => Some(usize::MAX), + 1..=TOP_DOWN_DEPTH_LIMIT => Some(1), + _ => None, + } + } + + fn on_inline_success( + &mut self, callsite: &CallSite<'tcx>, - mut callee_body: Body<'tcx>, + caller_body: &mut Body<'tcx>, + new_blocks: std::ops::Range, ) { - let terminator = caller_body[callsite.block].terminator.take().unwrap(); - let TerminatorKind::Call { func, args, destination, unwind, target, .. } = terminator.kind - else { - bug!("unexpected terminator kind {:?}", terminator.kind); - }; + self.changed = true; - let return_block = if let Some(block) = target { - // Prepare a new block for code that should execute when call returns. We don't use - // target block directly since it might have other predecessors. - let mut data = BasicBlockData::new(Some(Terminator { - source_info: terminator.source_info, - kind: TerminatorKind::Goto { target: block }, - })); - data.is_cleanup = caller_body[block].is_cleanup; - Some(caller_body.basic_blocks_mut().push(data)) - } else { - None + self.history.push(callsite.callee.def_id()); + process_blocks(self, caller_body, new_blocks); + self.history.pop(); + } + + fn on_inline_limit_reached(&self) -> bool { + true + } + + fn on_inline_failure(&self, _: &CallSite<'tcx>, _: &'static str) {} +} + +fn inline<'tcx, T: Inliner<'tcx>>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> bool { + let def_id = body.source.def_id(); + + // Only do inlining into fn bodies. + if !tcx.hir().body_owner_kind(def_id).is_fn_or_closure() { + return false; + } + + let mut inliner = T::new(tcx, def_id, body); + if !inliner.check_caller_mir_body(body) { + return false; + } + + let blocks = START_BLOCK..body.basic_blocks.next_index(); + process_blocks(&mut inliner, body, blocks); + inliner.changed() +} + +fn process_blocks<'tcx, I: Inliner<'tcx>>( + inliner: &mut I, + caller_body: &mut Body<'tcx>, + blocks: Range, +) { + let Some(inline_limit) = inliner.inline_limit_for_block() else { return }; + let mut inlined_count = 0; + for bb in blocks { + let bb_data = &caller_body[bb]; + if bb_data.is_cleanup { + continue; + } + + let Some(callsite) = resolve_callsite(inliner, caller_body, bb, bb_data) else { + continue; }; - // If the call is something like `a[*i] = f(i)`, where - // `i : &mut usize`, then just duplicating the `a[*i]` - // Place could result in two different locations if `f` - // writes to `i`. To prevent this we need to create a temporary - // borrow of the place and pass the destination as `*temp` instead. - fn dest_needs_borrow(place: Place<'_>) -> bool { - for elem in place.projection.iter() { - match elem { - ProjectionElem::Deref | ProjectionElem::Index(_) => return true, - _ => {} + let span = trace_span!("process_blocks", %callsite.callee, ?bb); + let _guard = span.enter(); + + match try_inlining(inliner, caller_body, &callsite) { + Err(reason) => { + debug!("not-inlined {} [{}]", callsite.callee, reason); + inliner.on_inline_failure(&callsite, reason); + } + Ok(new_blocks) => { + debug!("inlined {}", callsite.callee); + inliner.on_inline_success(&callsite, caller_body, new_blocks); + + inlined_count += 1; + if inlined_count == inline_limit { + if inliner.on_inline_limit_reached() { + return; + } } } + } + } +} + +fn resolve_callsite<'tcx, I: Inliner<'tcx>>( + inliner: &I, + caller_body: &Body<'tcx>, + bb: BasicBlock, + bb_data: &BasicBlockData<'tcx>, +) -> Option> { + let tcx = inliner.tcx(); + // Only consider direct calls to functions + let terminator = bb_data.terminator(); + + // FIXME(explicit_tail_calls): figure out if we can inline tail calls + if let TerminatorKind::Call { ref func, fn_span, .. } = terminator.kind { + let func_ty = func.ty(caller_body, tcx); + if let ty::FnDef(def_id, args) = *func_ty.kind() { + if !inliner.should_inline_for_callee(def_id) { + debug!("not enabled"); + return None; + } + + // To resolve an instance its args have to be fully normalized. + let args = tcx.try_normalize_erasing_regions(inliner.typing_env(), args).ok()?; + let callee = + Instance::try_resolve(tcx, inliner.typing_env(), def_id, args).ok().flatten()?; + + if let InstanceKind::Virtual(..) | InstanceKind::Intrinsic(_) = callee.def { + return None; + } - false + if inliner.history().contains(&callee.def_id()) { + return None; + } + + let fn_sig = tcx.fn_sig(def_id).instantiate(tcx, args); + + // Additionally, check that the body that we're inlining actually agrees + // with the ABI of the trait that the item comes from. + if let InstanceKind::Item(instance_def_id) = callee.def + && tcx.def_kind(instance_def_id) == DefKind::AssocFn + && let instance_fn_sig = tcx.fn_sig(instance_def_id).skip_binder() + && instance_fn_sig.abi() != fn_sig.abi() + { + return None; + } + + let source_info = SourceInfo { span: fn_span, ..terminator.source_info }; + + return Some(CallSite { callee, fn_sig, block: bb, source_info }); } + } - let dest = if dest_needs_borrow(destination) { - trace!("creating temp for return destination"); - let dest = Rvalue::Ref( - self.tcx.lifetimes.re_erased, - BorrowKind::Mut { kind: MutBorrowKind::Default }, - destination, - ); - let dest_ty = dest.ty(caller_body, self.tcx); - let temp = - Place::from(self.new_call_temp(caller_body, callsite, dest_ty, return_block)); - caller_body[callsite.block].statements.push(Statement { - source_info: callsite.source_info, - kind: StatementKind::Assign(Box::new((temp, dest))), - }); - self.tcx.mk_place_deref(temp) - } else { - destination - }; + None +} - // Always create a local to hold the destination, as `RETURN_PLACE` may appear - // where a full `Place` is not allowed. - let (remap_destination, destination_local) = if let Some(d) = dest.as_local() { - (false, d) - } else { - ( - true, - self.new_call_temp( - caller_body, - callsite, - destination.ty(caller_body, self.tcx).ty, - return_block, - ), - ) - }; +/// Attempts to inline a callsite into the caller body. When successful returns basic blocks +/// containing the inlined body. Otherwise returns an error describing why inlining didn't take +/// place. +fn try_inlining<'tcx, I: Inliner<'tcx>>( + inliner: &I, + caller_body: &mut Body<'tcx>, + callsite: &CallSite<'tcx>, +) -> Result, &'static str> { + let tcx = inliner.tcx(); + check_mir_is_available(inliner, caller_body, callsite.callee)?; + + let callee_attrs = tcx.codegen_fn_attrs(callsite.callee.def_id()); + let cross_crate_inlinable = tcx.cross_crate_inlinable(callsite.callee.def_id()); + check_codegen_attributes(inliner, callsite, callee_attrs, cross_crate_inlinable)?; + + // Intrinsic fallback bodies are automatically made cross-crate inlineable, + // but at this stage we don't know whether codegen knows the intrinsic, + // so just conservatively don't inline it. + if tcx.has_attr(callsite.callee.def_id(), sym::rustc_intrinsic) { + return Err("callee is an intrinsic"); + } - // Copy the arguments if needed. - let args = self.make_call_args(args, callsite, caller_body, &callee_body, return_block); + let terminator = caller_body[callsite.block].terminator.as_ref().unwrap(); + let TerminatorKind::Call { args, destination, .. } = &terminator.kind else { bug!() }; + let destination_ty = destination.ty(&caller_body.local_decls, tcx).ty; + for arg in args { + if !arg.node.ty(&caller_body.local_decls, tcx).is_sized(tcx, inliner.typing_env()) { + // We do not allow inlining functions with unsized params. Inlining these functions + // could create unsized locals, which are unsound and being phased out. + return Err("call has unsized argument"); + } + } - let mut integrator = Integrator { - args: &args, - new_locals: Local::new(caller_body.local_decls.len()).., - new_scopes: SourceScope::new(caller_body.source_scopes.len()).., - new_blocks: BasicBlock::new(caller_body.basic_blocks.len()).., - destination: destination_local, - callsite_scope: caller_body.source_scopes[callsite.source_info.scope].clone(), - callsite, - cleanup_block: unwind, - in_cleanup_block: false, - return_block, - tcx: self.tcx, - always_live_locals: BitSet::new_filled(callee_body.local_decls.len()), + let callee_body = try_instance_mir(tcx, callsite.callee.def)?; + inliner.check_callee_mir_body(callsite, callee_body, callee_attrs, cross_crate_inlinable)?; + + let Ok(callee_body) = callsite.callee.try_instantiate_mir_and_normalize_erasing_regions( + tcx, + inliner.typing_env(), + ty::EarlyBinder::bind(callee_body.clone()), + ) else { + debug!("failed to normalize callee body"); + return Err("implementation limitation"); + }; + + // Normally, this shouldn't be required, but trait normalization failure can create a + // validation ICE. + if !validate_types(tcx, inliner.typing_env(), &callee_body, &caller_body).is_empty() { + debug!("failed to validate callee body"); + return Err("implementation limitation"); + } + + // Check call signature compatibility. + // Normally, this shouldn't be required, but trait normalization failure can create a + // validation ICE. + let output_type = callee_body.return_ty(); + if !util::sub_types(tcx, inliner.typing_env(), output_type, destination_ty) { + trace!(?output_type, ?destination_ty); + debug!("failed to normalize return type"); + return Err("implementation limitation"); + } + if callsite.fn_sig.abi() == ExternAbi::RustCall { + // FIXME: Don't inline user-written `extern "rust-call"` functions, + // since this is generally perf-negative on rustc, and we hope that + // LLVM will inline these functions instead. + if callee_body.spread_arg.is_some() { + return Err("user-written rust-call functions"); + } + + let (self_arg, arg_tuple) = match &args[..] { + [arg_tuple] => (None, arg_tuple), + [self_arg, arg_tuple] => (Some(self_arg), arg_tuple), + _ => bug!("Expected `rust-call` to have 1 or 2 args"), }; - // Map all `Local`s, `SourceScope`s and `BasicBlock`s to new ones - // (or existing ones, in a few special cases) in the caller. - integrator.visit_body(&mut callee_body); + let self_arg_ty = self_arg.map(|self_arg| self_arg.node.ty(&caller_body.local_decls, tcx)); - // If there are any locals without storage markers, give them storage only for the - // duration of the call. - for local in callee_body.vars_and_temps_iter() { - if integrator.always_live_locals.contains(local) { - let new_local = integrator.map_local(local); - caller_body[callsite.block].statements.push(Statement { - source_info: callsite.source_info, - kind: StatementKind::StorageLive(new_local), - }); + let arg_tuple_ty = arg_tuple.node.ty(&caller_body.local_decls, tcx); + let ty::Tuple(arg_tuple_tys) = *arg_tuple_ty.kind() else { + bug!("Closure arguments are not passed as a tuple"); + }; + + for (arg_ty, input) in + self_arg_ty.into_iter().chain(arg_tuple_tys).zip(callee_body.args_iter()) + { + let input_type = callee_body.local_decls[input].ty; + if !util::sub_types(tcx, inliner.typing_env(), input_type, arg_ty) { + trace!(?arg_ty, ?input_type); + debug!("failed to normalize tuple argument type"); + return Err("implementation limitation"); } } - if let Some(block) = return_block { - // To avoid repeated O(n) insert, push any new statements to the end and rotate - // the slice once. - let mut n = 0; - if remap_destination { - caller_body[block].statements.push(Statement { - source_info: callsite.source_info, - kind: StatementKind::Assign(Box::new(( - dest, - Rvalue::Use(Operand::Move(destination_local.into())), - ))), - }); - n += 1; + } else { + for (arg, input) in args.iter().zip(callee_body.args_iter()) { + let input_type = callee_body.local_decls[input].ty; + let arg_ty = arg.node.ty(&caller_body.local_decls, tcx); + if !util::sub_types(tcx, inliner.typing_env(), input_type, arg_ty) { + trace!(?arg_ty, ?input_type); + debug!("failed to normalize argument type"); + return Err("implementation limitation"); } - for local in callee_body.vars_and_temps_iter().rev() { - if integrator.always_live_locals.contains(local) { - let new_local = integrator.map_local(local); - caller_body[block].statements.push(Statement { - source_info: callsite.source_info, - kind: StatementKind::StorageDead(new_local), - }); - n += 1; - } + } + } + + let old_blocks = caller_body.basic_blocks.next_index(); + inline_call(inliner, caller_body, callsite, callee_body); + let new_blocks = old_blocks..caller_body.basic_blocks.next_index(); + + Ok(new_blocks) +} + +fn check_mir_is_available<'tcx, I: Inliner<'tcx>>( + inliner: &I, + caller_body: &Body<'tcx>, + callee: Instance<'tcx>, +) -> Result<(), &'static str> { + let caller_def_id = caller_body.source.def_id(); + let callee_def_id = callee.def_id(); + if callee_def_id == caller_def_id { + return Err("self-recursion"); + } + + match callee.def { + InstanceKind::Item(_) => { + // If there is no MIR available (either because it was not in metadata or + // because it has no MIR because it's an extern function), then the inliner + // won't cause cycles on this. + if !inliner.tcx().is_mir_available(callee_def_id) { + debug!("item MIR unavailable"); + return Err("implementation limitation"); } - caller_body[block].statements.rotate_right(n); + } + // These have no own callable MIR. + InstanceKind::Intrinsic(_) | InstanceKind::Virtual(..) => { + debug!("instance without MIR (intrinsic / virtual)"); + return Err("implementation limitation"); } - // Insert all of the (mapped) parts of the callee body into the caller. - caller_body.local_decls.extend(callee_body.drain_vars_and_temps()); - caller_body.source_scopes.append(&mut callee_body.source_scopes); - if self - .tcx - .sess - .opts - .unstable_opts - .inline_mir_preserve_debug - .unwrap_or(self.tcx.sess.opts.debuginfo != DebugInfo::None) - { - // Note that we need to preserve these in the standard library so that - // people working on rust can build with or without debuginfo while - // still getting consistent results from the mir-opt tests. - caller_body.var_debug_info.append(&mut callee_body.var_debug_info); + // FIXME(#127030): `ConstParamHasTy` has bad interactions with + // the drop shim builder, which does not evaluate predicates in + // the correct param-env for types being dropped. Stall resolving + // the MIR for this instance until all of its const params are + // substituted. + InstanceKind::DropGlue(_, Some(ty)) if ty.has_type_flags(TypeFlags::HAS_CT_PARAM) => { + debug!("still needs substitution"); + return Err("implementation limitation"); } - caller_body.basic_blocks_mut().append(callee_body.basic_blocks_mut()); - caller_body[callsite.block].terminator = Some(Terminator { - source_info: callsite.source_info, - kind: TerminatorKind::Goto { target: integrator.map_block(START_BLOCK) }, - }); + // This cannot result in an immediate cycle since the callee MIR is a shim, which does + // not get any optimizations run on it. Any subsequent inlining may cause cycles, but we + // do not need to catch this here, we can wait until the inliner decides to continue + // inlining a second time. + InstanceKind::VTableShim(_) + | InstanceKind::ReifyShim(..) + | InstanceKind::FnPtrShim(..) + | InstanceKind::ClosureOnceShim { .. } + | InstanceKind::ConstructCoroutineInClosureShim { .. } + | InstanceKind::DropGlue(..) + | InstanceKind::CloneShim(..) + | InstanceKind::ThreadLocalShim(..) + | InstanceKind::FnPtrAddrShim(..) + | InstanceKind::AsyncDropGlueCtorShim(..) => return Ok(()), + } - // Copy required constants from the callee_body into the caller_body. Although we are only - // pushing unevaluated consts to `required_consts`, here they may have been evaluated - // because we are calling `instantiate_and_normalize_erasing_regions` -- so we filter again. - caller_body.required_consts.as_mut().unwrap().extend( - callee_body.required_consts().into_iter().filter(|ct| ct.const_.is_required_const()), - ); - // Now that we incorporated the callee's `required_consts`, we can remove the callee from - // `mentioned_items` -- but we have to take their `mentioned_items` in return. This does - // some extra work here to save the monomorphization collector work later. It helps a lot, - // since monomorphization can avoid a lot of work when the "mentioned items" are similar to - // the actually used items. By doing this we can entirely avoid visiting the callee! - // We need to reconstruct the `required_item` for the callee so that we can find and - // remove it. - let callee_item = MentionedItem::Fn(func.ty(caller_body, self.tcx)); - let caller_mentioned_items = caller_body.mentioned_items.as_mut().unwrap(); - if let Some(idx) = caller_mentioned_items.iter().position(|item| item.node == callee_item) { - // We found the callee, so remove it and add its items instead. - caller_mentioned_items.remove(idx); - caller_mentioned_items.extend(callee_body.mentioned_items()); - } else { - // If we can't find the callee, there's no point in adding its items. Probably it - // already got removed by being inlined elsewhere in the same function, so we already - // took its items. + if inliner.tcx().is_constructor(callee_def_id) { + trace!("constructors always have MIR"); + // Constructor functions cannot cause a query cycle. + return Ok(()); + } + + if callee_def_id.is_local() + && !inliner + .tcx() + .is_lang_item(inliner.tcx().parent(caller_def_id), rustc_hir::LangItem::FnOnce) + { + // If we know for sure that the function we're calling will itself try to + // call us, then we avoid inlining that function. + if inliner.tcx().mir_callgraph_reachable((callee, caller_def_id.expect_local())) { + debug!("query cycle avoidance"); + return Err("caller might be reachable from callee"); } + + Ok(()) + } else { + // This cannot result in an immediate cycle since the callee MIR is from another crate + // and is already optimized. Any subsequent inlining may cause cycles, but we do + // not need to catch this here, we can wait until the inliner decides to continue + // inlining a second time. + trace!("functions from other crates always have MIR"); + Ok(()) } +} - fn make_call_args( - &self, - args: Box<[Spanned>]>, - callsite: &CallSite<'tcx>, - caller_body: &mut Body<'tcx>, - callee_body: &Body<'tcx>, - return_block: Option, - ) -> Box<[Local]> { - let tcx = self.tcx; - - // There is a bit of a mismatch between the *caller* of a closure and the *callee*. - // The caller provides the arguments wrapped up in a tuple: - // - // tuple_tmp = (a, b, c) - // Fn::call(closure_ref, tuple_tmp) - // - // meanwhile the closure body expects the arguments (here, `a`, `b`, and `c`) - // as distinct arguments. (This is the "rust-call" ABI hack.) Normally, codegen has - // the job of unpacking this tuple. But here, we are codegen. =) So we want to create - // a vector like - // - // [closure_ref, tuple_tmp.0, tuple_tmp.1, tuple_tmp.2] - // - // Except for one tiny wrinkle: we don't actually want `tuple_tmp.0`. It's more convenient - // if we "spill" that into *another* temporary, so that we can map the argument - // variable in the callee MIR directly to an argument variable on our side. - // So we introduce temporaries like: - // - // tmp0 = tuple_tmp.0 - // tmp1 = tuple_tmp.1 - // tmp2 = tuple_tmp.2 - // - // and the vector is `[closure_ref, tmp0, tmp1, tmp2]`. - if callsite.fn_sig.abi() == ExternAbi::RustCall && callee_body.spread_arg.is_none() { - // FIXME(edition_2024): switch back to a normal method call. - let mut args = <_>::into_iter(args); - let self_ = self.create_temp_if_necessary( - args.next().unwrap().node, - callsite, - caller_body, - return_block, - ); - let tuple = self.create_temp_if_necessary( - args.next().unwrap().node, - callsite, - caller_body, - return_block, - ); - assert!(args.next().is_none()); - - let tuple = Place::from(tuple); - let ty::Tuple(tuple_tys) = tuple.ty(caller_body, tcx).ty.kind() else { - bug!("Closure arguments are not passed as a tuple"); - }; +/// Returns an error if inlining is not possible based on codegen attributes alone. A success +/// indicates that inlining decision should be based on other criteria. +fn check_codegen_attributes<'tcx, I: Inliner<'tcx>>( + inliner: &I, + callsite: &CallSite<'tcx>, + callee_attrs: &CodegenFnAttrs, + cross_crate_inlinable: bool, +) -> Result<(), &'static str> { + let tcx = inliner.tcx(); + if tcx.has_attr(callsite.callee.def_id(), sym::rustc_no_mir_inline) { + return Err("#[rustc_no_mir_inline]"); + } - // The `closure_ref` in our example above. - let closure_ref_arg = iter::once(self_); + if let InlineAttr::Never = callee_attrs.inline { + return Err("never inline attribute"); + } - // The `tmp0`, `tmp1`, and `tmp2` in our example above. - let tuple_tmp_args = tuple_tys.iter().enumerate().map(|(i, ty)| { - // This is e.g., `tuple_tmp.0` in our example above. - let tuple_field = Operand::Move(tcx.mk_place_field(tuple, FieldIdx::new(i), ty)); + // FIXME(#127234): Coverage instrumentation currently doesn't handle inlined + // MIR correctly when Modified Condition/Decision Coverage is enabled. + if tcx.sess.instrument_coverage_mcdc() { + return Err("incompatible with MC/DC coverage"); + } - // Spill to a local to make e.g., `tmp0`. - self.create_temp_if_necessary(tuple_field, callsite, caller_body, return_block) - }); + // Reachability pass defines which functions are eligible for inlining. Generally inlining + // other functions is incorrect because they could reference symbols that aren't exported. + let is_generic = callsite.callee.args.non_erasable_generics().next().is_some(); + if !is_generic && !cross_crate_inlinable { + return Err("not exported"); + } - closure_ref_arg.chain(tuple_tmp_args).collect() - } else { - // FIXME(edition_2024): switch back to a normal method call. - <_>::into_iter(args) - .map(|a| self.create_temp_if_necessary(a.node, callsite, caller_body, return_block)) - .collect() - } + if callsite.fn_sig.c_variadic() { + return Err("C variadic"); } - /// If `arg` is already a temporary, returns it. Otherwise, introduces a fresh - /// temporary `T` and an instruction `T = arg`, and returns `T`. - fn create_temp_if_necessary( - &self, - arg: Operand<'tcx>, - callsite: &CallSite<'tcx>, - caller_body: &mut Body<'tcx>, - return_block: Option, - ) -> Local { - // Reuse the operand if it is a moved temporary. - if let Operand::Move(place) = &arg - && let Some(local) = place.as_local() - && caller_body.local_kind(local) == LocalKind::Temp - { - return local; - } + if callee_attrs.flags.contains(CodegenFnAttrFlags::COLD) { + return Err("cold"); + } - // Otherwise, create a temporary for the argument. - trace!("creating temp for argument {:?}", arg); - let arg_ty = arg.ty(caller_body, self.tcx); - let local = self.new_call_temp(caller_body, callsite, arg_ty, return_block); - caller_body[callsite.block].statements.push(Statement { - source_info: callsite.source_info, - kind: StatementKind::Assign(Box::new((Place::from(local), Rvalue::Use(arg)))), - }); - local + let codegen_fn_attrs = tcx.codegen_fn_attrs(inliner.caller_def_id()); + if callee_attrs.no_sanitize != codegen_fn_attrs.no_sanitize { + return Err("incompatible sanitizer set"); } - /// Introduces a new temporary into the caller body that is live for the duration of the call. - fn new_call_temp( - &self, - caller_body: &mut Body<'tcx>, - callsite: &CallSite<'tcx>, - ty: Ty<'tcx>, - return_block: Option, - ) -> Local { - let local = caller_body.local_decls.push(LocalDecl::new(ty, callsite.source_info.span)); + // Two functions are compatible if the callee has no attribute (meaning + // that it's codegen agnostic), or sets an attribute that is identical + // to this function's attribute. + if callee_attrs.instruction_set.is_some() + && callee_attrs.instruction_set != codegen_fn_attrs.instruction_set + { + return Err("incompatible instruction set"); + } + + let callee_feature_names = callee_attrs.target_features.iter().map(|f| f.name); + let this_feature_names = codegen_fn_attrs.target_features.iter().map(|f| f.name); + if callee_feature_names.ne(this_feature_names) { + // In general it is not correct to inline a callee with target features that are a + // subset of the caller. This is because the callee might contain calls, and the ABI of + // those calls depends on the target features of the surrounding function. By moving a + // `Call` terminator from one MIR body to another with more target features, we might + // change the ABI of that call! + return Err("incompatible target features"); + } + + Ok(()) +} +fn inline_call<'tcx, I: Inliner<'tcx>>( + inliner: &I, + caller_body: &mut Body<'tcx>, + callsite: &CallSite<'tcx>, + mut callee_body: Body<'tcx>, +) { + let tcx = inliner.tcx(); + let terminator = caller_body[callsite.block].terminator.take().unwrap(); + let TerminatorKind::Call { func, args, destination, unwind, target, .. } = terminator.kind + else { + bug!("unexpected terminator kind {:?}", terminator.kind); + }; + + let return_block = if let Some(block) = target { + // Prepare a new block for code that should execute when call returns. We don't use + // target block directly since it might have other predecessors. + let mut data = BasicBlockData::new(Some(Terminator { + source_info: terminator.source_info, + kind: TerminatorKind::Goto { target: block }, + })); + data.is_cleanup = caller_body[block].is_cleanup; + Some(caller_body.basic_blocks_mut().push(data)) + } else { + None + }; + + // If the call is something like `a[*i] = f(i)`, where + // `i : &mut usize`, then just duplicating the `a[*i]` + // Place could result in two different locations if `f` + // writes to `i`. To prevent this we need to create a temporary + // borrow of the place and pass the destination as `*temp` instead. + fn dest_needs_borrow(place: Place<'_>) -> bool { + for elem in place.projection.iter() { + match elem { + ProjectionElem::Deref | ProjectionElem::Index(_) => return true, + _ => {} + } + } + + false + } + + let dest = if dest_needs_borrow(destination) { + trace!("creating temp for return destination"); + let dest = Rvalue::Ref( + tcx.lifetimes.re_erased, + BorrowKind::Mut { kind: MutBorrowKind::Default }, + destination, + ); + let dest_ty = dest.ty(caller_body, tcx); + let temp = Place::from(new_call_temp(caller_body, callsite, dest_ty, return_block)); caller_body[callsite.block].statements.push(Statement { source_info: callsite.source_info, - kind: StatementKind::StorageLive(local), + kind: StatementKind::Assign(Box::new((temp, dest))), }); + tcx.mk_place_deref(temp) + } else { + destination + }; + + // Always create a local to hold the destination, as `RETURN_PLACE` may appear + // where a full `Place` is not allowed. + let (remap_destination, destination_local) = if let Some(d) = dest.as_local() { + (false, d) + } else { + ( + true, + new_call_temp(caller_body, callsite, destination.ty(caller_body, tcx).ty, return_block), + ) + }; - if let Some(block) = return_block { - caller_body[block].statements.insert(0, Statement { + // Copy the arguments if needed. + let args = make_call_args(inliner, args, callsite, caller_body, &callee_body, return_block); + + let mut integrator = Integrator { + args: &args, + new_locals: Local::new(caller_body.local_decls.len()).., + new_scopes: SourceScope::new(caller_body.source_scopes.len()).., + new_blocks: BasicBlock::new(caller_body.basic_blocks.len()).., + destination: destination_local, + callsite_scope: caller_body.source_scopes[callsite.source_info.scope].clone(), + callsite, + cleanup_block: unwind, + in_cleanup_block: false, + return_block, + tcx, + always_live_locals: BitSet::new_filled(callee_body.local_decls.len()), + }; + + // Map all `Local`s, `SourceScope`s and `BasicBlock`s to new ones + // (or existing ones, in a few special cases) in the caller. + integrator.visit_body(&mut callee_body); + + // If there are any locals without storage markers, give them storage only for the + // duration of the call. + for local in callee_body.vars_and_temps_iter() { + if integrator.always_live_locals.contains(local) { + let new_local = integrator.map_local(local); + caller_body[callsite.block].statements.push(Statement { source_info: callsite.source_info, - kind: StatementKind::StorageDead(local), + kind: StatementKind::StorageLive(new_local), }); } + } + if let Some(block) = return_block { + // To avoid repeated O(n) insert, push any new statements to the end and rotate + // the slice once. + let mut n = 0; + if remap_destination { + caller_body[block].statements.push(Statement { + source_info: callsite.source_info, + kind: StatementKind::Assign(Box::new(( + dest, + Rvalue::Use(Operand::Move(destination_local.into())), + ))), + }); + n += 1; + } + for local in callee_body.vars_and_temps_iter().rev() { + if integrator.always_live_locals.contains(local) { + let new_local = integrator.map_local(local); + caller_body[block].statements.push(Statement { + source_info: callsite.source_info, + kind: StatementKind::StorageDead(new_local), + }); + n += 1; + } + } + caller_body[block].statements.rotate_right(n); + } + + // Insert all of the (mapped) parts of the callee body into the caller. + caller_body.local_decls.extend(callee_body.drain_vars_and_temps()); + caller_body.source_scopes.append(&mut callee_body.source_scopes); + if tcx + .sess + .opts + .unstable_opts + .inline_mir_preserve_debug + .unwrap_or(tcx.sess.opts.debuginfo != DebugInfo::None) + { + // Note that we need to preserve these in the standard library so that + // people working on rust can build with or without debuginfo while + // still getting consistent results from the mir-opt tests. + caller_body.var_debug_info.append(&mut callee_body.var_debug_info); + } + caller_body.basic_blocks_mut().append(callee_body.basic_blocks_mut()); + + caller_body[callsite.block].terminator = Some(Terminator { + source_info: callsite.source_info, + kind: TerminatorKind::Goto { target: integrator.map_block(START_BLOCK) }, + }); + + // Copy required constants from the callee_body into the caller_body. Although we are only + // pushing unevaluated consts to `required_consts`, here they may have been evaluated + // because we are calling `instantiate_and_normalize_erasing_regions` -- so we filter again. + caller_body.required_consts.as_mut().unwrap().extend( + callee_body.required_consts().into_iter().filter(|ct| ct.const_.is_required_const()), + ); + // Now that we incorporated the callee's `required_consts`, we can remove the callee from + // `mentioned_items` -- but we have to take their `mentioned_items` in return. This does + // some extra work here to save the monomorphization collector work later. It helps a lot, + // since monomorphization can avoid a lot of work when the "mentioned items" are similar to + // the actually used items. By doing this we can entirely avoid visiting the callee! + // We need to reconstruct the `required_item` for the callee so that we can find and + // remove it. + let callee_item = MentionedItem::Fn(func.ty(caller_body, tcx)); + let caller_mentioned_items = caller_body.mentioned_items.as_mut().unwrap(); + if let Some(idx) = caller_mentioned_items.iter().position(|item| item.node == callee_item) { + // We found the callee, so remove it and add its items instead. + caller_mentioned_items.remove(idx); + caller_mentioned_items.extend(callee_body.mentioned_items()); + } else { + // If we can't find the callee, there's no point in adding its items. Probably it + // already got removed by being inlined elsewhere in the same function, so we already + // took its items. + } +} + +fn make_call_args<'tcx, I: Inliner<'tcx>>( + inliner: &I, + args: Box<[Spanned>]>, + callsite: &CallSite<'tcx>, + caller_body: &mut Body<'tcx>, + callee_body: &Body<'tcx>, + return_block: Option, +) -> Box<[Local]> { + let tcx = inliner.tcx(); + + // There is a bit of a mismatch between the *caller* of a closure and the *callee*. + // The caller provides the arguments wrapped up in a tuple: + // + // tuple_tmp = (a, b, c) + // Fn::call(closure_ref, tuple_tmp) + // + // meanwhile the closure body expects the arguments (here, `a`, `b`, and `c`) + // as distinct arguments. (This is the "rust-call" ABI hack.) Normally, codegen has + // the job of unpacking this tuple. But here, we are codegen. =) So we want to create + // a vector like + // + // [closure_ref, tuple_tmp.0, tuple_tmp.1, tuple_tmp.2] + // + // Except for one tiny wrinkle: we don't actually want `tuple_tmp.0`. It's more convenient + // if we "spill" that into *another* temporary, so that we can map the argument + // variable in the callee MIR directly to an argument variable on our side. + // So we introduce temporaries like: + // + // tmp0 = tuple_tmp.0 + // tmp1 = tuple_tmp.1 + // tmp2 = tuple_tmp.2 + // + // and the vector is `[closure_ref, tmp0, tmp1, tmp2]`. + if callsite.fn_sig.abi() == ExternAbi::RustCall && callee_body.spread_arg.is_none() { + // FIXME(edition_2024): switch back to a normal method call. + let mut args = <_>::into_iter(args); + let self_ = create_temp_if_necessary( + inliner, + args.next().unwrap().node, + callsite, + caller_body, + return_block, + ); + let tuple = create_temp_if_necessary( + inliner, + args.next().unwrap().node, + callsite, + caller_body, + return_block, + ); + assert!(args.next().is_none()); + + let tuple = Place::from(tuple); + let ty::Tuple(tuple_tys) = tuple.ty(caller_body, tcx).ty.kind() else { + bug!("Closure arguments are not passed as a tuple"); + }; + + // The `closure_ref` in our example above. + let closure_ref_arg = iter::once(self_); + + // The `tmp0`, `tmp1`, and `tmp2` in our example above. + let tuple_tmp_args = tuple_tys.iter().enumerate().map(|(i, ty)| { + // This is e.g., `tuple_tmp.0` in our example above. + let tuple_field = Operand::Move(tcx.mk_place_field(tuple, FieldIdx::new(i), ty)); + + // Spill to a local to make e.g., `tmp0`. + create_temp_if_necessary(inliner, tuple_field, callsite, caller_body, return_block) + }); + + closure_ref_arg.chain(tuple_tmp_args).collect() + } else { + // FIXME(edition_2024): switch back to a normal method call. + <_>::into_iter(args) + .map(|a| create_temp_if_necessary(inliner, a.node, callsite, caller_body, return_block)) + .collect() + } +} - local +/// If `arg` is already a temporary, returns it. Otherwise, introduces a fresh temporary `T` and an +/// instruction `T = arg`, and returns `T`. +fn create_temp_if_necessary<'tcx, I: Inliner<'tcx>>( + inliner: &I, + arg: Operand<'tcx>, + callsite: &CallSite<'tcx>, + caller_body: &mut Body<'tcx>, + return_block: Option, +) -> Local { + // Reuse the operand if it is a moved temporary. + if let Operand::Move(place) = &arg + && let Some(local) = place.as_local() + && caller_body.local_kind(local) == LocalKind::Temp + { + return local; + } + + // Otherwise, create a temporary for the argument. + trace!("creating temp for argument {:?}", arg); + let arg_ty = arg.ty(caller_body, inliner.tcx()); + let local = new_call_temp(caller_body, callsite, arg_ty, return_block); + caller_body[callsite.block].statements.push(Statement { + source_info: callsite.source_info, + kind: StatementKind::Assign(Box::new((Place::from(local), Rvalue::Use(arg)))), + }); + local +} + +/// Introduces a new temporary into the caller body that is live for the duration of the call. +fn new_call_temp<'tcx>( + caller_body: &mut Body<'tcx>, + callsite: &CallSite<'tcx>, + ty: Ty<'tcx>, + return_block: Option, +) -> Local { + let local = caller_body.local_decls.push(LocalDecl::new(ty, callsite.source_info.span)); + + caller_body[callsite.block].statements.push(Statement { + source_info: callsite.source_info, + kind: StatementKind::StorageLive(local), + }); + + if let Some(block) = return_block { + caller_body[block].statements.insert(0, Statement { + source_info: callsite.source_info, + kind: StatementKind::StorageDead(local), + }); } + + local } /** diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs index f0fcb44603b15..6b2b7ee5241fe 100644 --- a/compiler/rustc_mir_transform/src/lib.rs +++ b/compiler/rustc_mir_transform/src/lib.rs @@ -141,7 +141,7 @@ declare_passes! { mod gvn : GVN; // Made public so that `mir_drops_elaborated_and_const_checked` can be overridden // by custom rustc drivers, running all the steps by themselves. See #114628. - pub mod inline : Inline; + pub mod inline : Inline, ForceInline; mod instsimplify : InstSimplify { BeforeInline, AfterSimplifyCfg }; mod jump_threading : JumpThreading; mod known_panics_lint : KnownPanicsLint; @@ -485,7 +485,9 @@ fn mir_drops_elaborated_and_const_checked(tcx: TyCtxt<'_>, def: LocalDefId) -> & let is_fn_like = tcx.def_kind(def).is_fn_like(); if is_fn_like { // Do not compute the mir call graph without said call graph actually being used. - if pm::should_run_pass(tcx, &inline::Inline) { + if pm::should_run_pass(tcx, &inline::Inline) + || inline::ForceInline::should_run_pass_for_callee(tcx, def.to_def_id()) + { tcx.ensure_with_value().mir_inliner_callees(ty::InstanceKind::Item(def.to_def_id())); } } @@ -662,6 +664,8 @@ fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { // Perform instsimplify before inline to eliminate some trivial calls (like clone // shims). &instsimplify::InstSimplify::BeforeInline, + // Perform inlining of `#[rustc_force_inline]`-annotated callees. + &inline::ForceInline, // Perform inlining, which may add a lot of code. &inline::Inline, // Code from other crates may have storage markers, so this needs to happen after diff --git a/compiler/rustc_mir_transform/src/pass_manager.rs b/compiler/rustc_mir_transform/src/pass_manager.rs index 8a45ce0762d59..c3f0a989ce105 100644 --- a/compiler/rustc_mir_transform/src/pass_manager.rs +++ b/compiler/rustc_mir_transform/src/pass_manager.rs @@ -79,6 +79,12 @@ pub(super) trait MirPass<'tcx> { true } + /// Returns `true` if this pass can be overridden by `-Zenable-mir-passes`. This should be + /// true for basically every pass other than those that are necessary for correctness. + fn can_be_overridden(&self) -> bool { + true + } + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>); fn is_mir_dump_enabled(&self) -> bool { @@ -176,6 +182,10 @@ where { let name = pass.name(); + if !pass.can_be_overridden() { + return pass.is_enabled(tcx.sess); + } + let overridden_passes = &tcx.sess.opts.unstable_opts.mir_enable_passes; let overridden = overridden_passes.iter().rev().find(|(s, _)| s == &*name).map(|(_name, polarity)| { diff --git a/compiler/rustc_mir_transform/src/shim.rs b/compiler/rustc_mir_transform/src/shim.rs index b8383e734e2ba..a707ea760e520 100644 --- a/compiler/rustc_mir_transform/src/shim.rs +++ b/compiler/rustc_mir_transform/src/shim.rs @@ -19,7 +19,7 @@ use rustc_span::{DUMMY_SP, Span}; use tracing::{debug, instrument}; use crate::{ - abort_unwinding_calls, add_call_guards, add_moves_for_packed_drops, deref_separator, + abort_unwinding_calls, add_call_guards, add_moves_for_packed_drops, deref_separator, inline, instsimplify, mentioned_items, pass_manager as pm, remove_noop_landing_pads, simplify, }; @@ -154,6 +154,8 @@ fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceKind<'tcx>) -> Body< &remove_noop_landing_pads::RemoveNoopLandingPads, &simplify::SimplifyCfg::MakeShim, &instsimplify::InstSimplify::BeforeInline, + // Perform inlining of `#[rustc_force_inline]`-annotated callees. + &inline::ForceInline, &abort_unwinding_calls::AbortUnwindingCalls, &add_call_guards::CriticalCallEdges, ], diff --git a/compiler/rustc_mir_transform/src/validate.rs b/compiler/rustc_mir_transform/src/validate.rs index 404fbb6b83937..f9803cd56ce8c 100644 --- a/compiler/rustc_mir_transform/src/validate.rs +++ b/compiler/rustc_mir_transform/src/validate.rs @@ -1,6 +1,7 @@ //! Validates the MIR to ensure that invariants are upheld. use rustc_abi::{ExternAbi, FIRST_VARIANT, Size}; +use rustc_attr::InlineAttr; use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_hir::LangItem; use rustc_index::IndexVec; @@ -366,7 +367,8 @@ impl<'a, 'tcx> Visitor<'tcx> for CfgChecker<'a, 'tcx> { self.check_edge(location, *target, EdgeKind::Normal); self.check_unwind_edge(location, *unwind); } - TerminatorKind::Call { args, .. } | TerminatorKind::TailCall { args, .. } => { + TerminatorKind::Call { func, args, .. } + | TerminatorKind::TailCall { func, args, .. } => { // FIXME(explicit_tail_calls): refactor this & add tail-call specific checks if let TerminatorKind::Call { target, unwind, destination, .. } = terminator.kind { if let Some(target) = target { @@ -419,6 +421,13 @@ impl<'a, 'tcx> Visitor<'tcx> for CfgChecker<'a, 'tcx> { } } } + + if let ty::FnDef(did, ..) = func.ty(&self.body.local_decls, self.tcx).kind() + && self.body.phase >= MirPhase::Runtime(RuntimePhase::Optimized) + && matches!(self.tcx.codegen_fn_attrs(did).inline, InlineAttr::Force { .. }) + { + self.fail(location, "`#[rustc_force_inline]`-annotated function not inlined"); + } } TerminatorKind::Assert { target, unwind, .. } => { self.check_edge(location, *target, EdgeKind::Normal); diff --git a/compiler/rustc_monomorphize/src/collector.rs b/compiler/rustc_monomorphize/src/collector.rs index 480d82c1a385b..cb0493df77fe0 100644 --- a/compiler/rustc_monomorphize/src/collector.rs +++ b/compiler/rustc_monomorphize/src/collector.rs @@ -960,6 +960,14 @@ fn should_codegen_locally<'tcx>(tcx: TyCtxtAt<'tcx>, instance: Instance<'tcx>) - return false; } + if tcx.def_kind(def_id).has_codegen_attrs() + && matches!(tcx.codegen_fn_attrs(def_id).inline, rustc_attr::InlineAttr::Force { .. }) + { + // `#[rustc_force_inline]` items should never be codegened. This should be caught by + // the MIR validator. + return false; + } + if def_id.is_local() { // Local items cannot be referred to locally without monomorphizing them locally. return true; diff --git a/compiler/rustc_passes/messages.ftl b/compiler/rustc_passes/messages.ftl index 7a0a518bb513a..dad69df41ea46 100644 --- a/compiler/rustc_passes/messages.ftl +++ b/compiler/rustc_passes/messages.ftl @@ -646,6 +646,14 @@ passes_rustc_allow_const_fn_unstable = passes_rustc_dirty_clean = attribute requires -Z query-dep-graph to be enabled +passes_rustc_force_inline = + attribute should be applied to a function + .label = not a function definition + +passes_rustc_force_inline_coro = + attribute cannot be applied to a `async`, `gen` or `async gen` function + .label = `async`, `gen` or `async gen` function + passes_rustc_layout_scalar_valid_range_arg = expected exactly one integer literal argument diff --git a/compiler/rustc_passes/src/check_attr.rs b/compiler/rustc_passes/src/check_attr.rs index 074fe77324faf..ae2d51f0aa1a7 100644 --- a/compiler/rustc_passes/src/check_attr.rs +++ b/compiler/rustc_passes/src/check_attr.rs @@ -249,7 +249,7 @@ impl<'tcx> CheckAttrVisitor<'tcx> { self.check_coroutine(attr, target); } [sym::linkage, ..] => self.check_linkage(attr, span, target), - [sym::rustc_pub_transparent, ..] => self.check_rustc_pub_transparent( attr.span, span, attrs), + [sym::rustc_pub_transparent, ..] => self.check_rustc_pub_transparent(attr.span, span, attrs), [ // ok sym::allow @@ -334,6 +334,7 @@ impl<'tcx> CheckAttrVisitor<'tcx> { self.check_repr(attrs, span, target, item, hir_id); self.check_used(attrs, target, span); + self.check_rustc_force_inline(hir_id, attrs, span, target); } fn inline_attr_str_error_with_macro_def(&self, hir_id: HirId, attr: &Attribute, sym: &str) { @@ -2424,6 +2425,45 @@ impl<'tcx> CheckAttrVisitor<'tcx> { } } + fn check_rustc_force_inline( + &self, + hir_id: HirId, + attrs: &[Attribute], + span: Span, + target: Target, + ) { + let force_inline_attr = attrs.iter().find(|attr| attr.has_name(sym::rustc_force_inline)); + match (target, force_inline_attr) { + (Target::Closure, None) => { + let is_coro = matches!( + self.tcx.hir().expect_expr(hir_id).kind, + hir::ExprKind::Closure(hir::Closure { + kind: hir::ClosureKind::Coroutine(..) + | hir::ClosureKind::CoroutineClosure(..), + .. + }) + ); + let parent_did = self.tcx.hir().get_parent_item(hir_id).to_def_id(); + let parent_span = self.tcx.def_span(parent_did); + let parent_force_inline_attr = + self.tcx.get_attr(parent_did, sym::rustc_force_inline); + if let Some(attr) = parent_force_inline_attr + && is_coro + { + self.dcx().emit_err(errors::RustcForceInlineCoro { + attr_span: attr.span, + span: parent_span, + }); + } + } + (Target::Fn, _) => (), + (_, Some(attr)) => { + self.dcx().emit_err(errors::RustcForceInline { attr_span: attr.span, span }); + } + (_, None) => (), + } + } + /// Checks if `#[autodiff]` is applied to an item other than a function item. fn check_autodiff(&self, _hir_id: HirId, _attr: &Attribute, span: Span, target: Target) { debug!("check_autodiff"); diff --git a/compiler/rustc_passes/src/errors.rs b/compiler/rustc_passes/src/errors.rs index fdc7e1bba2f0e..e63f9ed737a90 100644 --- a/compiler/rustc_passes/src/errors.rs +++ b/compiler/rustc_passes/src/errors.rs @@ -666,6 +666,24 @@ pub(crate) struct RustcPubTransparent { pub span: Span, } +#[derive(Diagnostic)] +#[diag(passes_rustc_force_inline)] +pub(crate) struct RustcForceInline { + #[primary_span] + pub attr_span: Span, + #[label] + pub span: Span, +} + +#[derive(Diagnostic)] +#[diag(passes_rustc_force_inline_coro)] +pub(crate) struct RustcForceInlineCoro { + #[primary_span] + pub attr_span: Span, + #[label] + pub span: Span, +} + #[derive(Diagnostic)] #[diag(passes_link_ordinal)] pub(crate) struct LinkOrdinal { diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index d30b17c9cd8dd..d5acb626fc425 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -1711,6 +1711,7 @@ symbols! { rustc_error, rustc_evaluate_where_clauses, rustc_expected_cgu_reuse, + rustc_force_inline, rustc_has_incoherent_inherent_impls, rustc_hidden_type_of_opaques, rustc_if_this_changed, diff --git a/compiler/rustc_trait_selection/messages.ftl b/compiler/rustc_trait_selection/messages.ftl index 1ab89ecde7a44..b82bb27eb795a 100644 --- a/compiler/rustc_trait_selection/messages.ftl +++ b/compiler/rustc_trait_selection/messages.ftl @@ -251,7 +251,9 @@ trait_selection_no_value_in_rustc_on_unimplemented = this attribute must have a trait_selection_nothing = {""} -trait_selection_oc_cant_coerce = cannot coerce intrinsics to function pointers +trait_selection_oc_cant_coerce_force_inline = + cannot coerce functions which must be inlined to function pointers +trait_selection_oc_cant_coerce_intrinsic = cannot coerce intrinsics to function pointers trait_selection_oc_closure_selfref = closure/coroutine type that references itself trait_selection_oc_const_compat = const not compatible with trait trait_selection_oc_fn_lang_correct_type = {$lang_item_name -> diff --git a/compiler/rustc_trait_selection/src/error_reporting/infer/mod.rs b/compiler/rustc_trait_selection/src/error_reporting/infer/mod.rs index f856a8d7abbe7..dac151c1adced 100644 --- a/compiler/rustc_trait_selection/src/error_reporting/infer/mod.rs +++ b/compiler/rustc_trait_selection/src/error_reporting/infer/mod.rs @@ -2214,7 +2214,7 @@ impl<'tcx> ObligationCause<'tcx> { { FailureCode::Error0644 } - TypeError::IntrinsicCast => FailureCode::Error0308, + TypeError::IntrinsicCast | TypeError::ForceInlineCast => FailureCode::Error0308, _ => FailureCode::Error0308, }, } @@ -2280,8 +2280,11 @@ impl<'tcx> ObligationCause<'tcx> { { ObligationCauseFailureCode::ClosureSelfref { span } } + TypeError::ForceInlineCast => { + ObligationCauseFailureCode::CantCoerceForceInline { span, subdiags } + } TypeError::IntrinsicCast => { - ObligationCauseFailureCode::CantCoerce { span, subdiags } + ObligationCauseFailureCode::CantCoerceIntrinsic { span, subdiags } } _ => ObligationCauseFailureCode::Generic { span, subdiags }, }, diff --git a/compiler/rustc_trait_selection/src/errors.rs b/compiler/rustc_trait_selection/src/errors.rs index 68fe90f0de29a..45c0622a8a66a 100644 --- a/compiler/rustc_trait_selection/src/errors.rs +++ b/compiler/rustc_trait_selection/src/errors.rs @@ -1730,8 +1730,15 @@ pub enum ObligationCauseFailureCode { #[primary_span] span: Span, }, - #[diag(trait_selection_oc_cant_coerce, code = E0308)] - CantCoerce { + #[diag(trait_selection_oc_cant_coerce_force_inline, code = E0308)] + CantCoerceForceInline { + #[primary_span] + span: Span, + #[subdiagnostic] + subdiags: Vec, + }, + #[diag(trait_selection_oc_cant_coerce_intrinsic, code = E0308)] + CantCoerceIntrinsic { #[primary_span] span: Span, #[subdiagnostic] diff --git a/compiler/rustc_type_ir/src/error.rs b/compiler/rustc_type_ir/src/error.rs index 55671b84dbc4f..68b11489ae7c1 100644 --- a/compiler/rustc_type_ir/src/error.rs +++ b/compiler/rustc_type_ir/src/error.rs @@ -51,6 +51,9 @@ pub enum TypeError { ConstMismatch(ExpectedFound), IntrinsicCast, + /// `#[rustc_force_inline]` functions must be inlined and must not be codegened independently, + /// so casting to a function pointer must be prohibited. + ForceInlineCast, /// Safe `#[target_feature]` functions are not assignable to safe function pointers. TargetFeatureCast(I::DefId), } @@ -83,6 +86,7 @@ impl TypeError { | ProjectionMismatched(_) | ExistentialMismatch(_) | ConstMismatch(_) + | ForceInlineCast | IntrinsicCast => true, } } diff --git a/tests/mir-opt/inline/forced.caller.ForceInline.diff b/tests/mir-opt/inline/forced.caller.ForceInline.diff new file mode 100644 index 0000000000000..7b70fd665666a --- /dev/null +++ b/tests/mir-opt/inline/forced.caller.ForceInline.diff @@ -0,0 +1,21 @@ +- // MIR for `caller` before ForceInline ++ // MIR for `caller` after ForceInline + + fn caller() -> () { + let mut _0: (); + let _1: (); ++ scope 1 (inlined callee_forced) { ++ } + + bb0: { + StorageLive(_1); +- _1 = callee_forced() -> [return: bb1, unwind continue]; +- } +- +- bb1: { + StorageDead(_1); + _0 = const (); + return; + } + } + diff --git a/tests/mir-opt/inline/forced.rs b/tests/mir-opt/inline/forced.rs new file mode 100644 index 0000000000000..66b75ff2bb953 --- /dev/null +++ b/tests/mir-opt/inline/forced.rs @@ -0,0 +1,13 @@ +// EMIT_MIT_FOR_EACH_PANIC_STRATEGY +//@ compile-flags: -Copt-level=0 --crate-type=lib +#![feature(rustc_attrs)] + +#[rustc_force_inline] +pub fn callee_forced() {} + +// EMIT_MIR forced.caller.ForceInline.diff +pub fn caller() { + callee_forced(); + // CHECK-LABEL: fn caller( + // CHECK: (inlined callee_forced) +} diff --git a/tests/mir-opt/inline/forced_async.caller.ForceInline.diff b/tests/mir-opt/inline/forced_async.caller.ForceInline.diff new file mode 100644 index 0000000000000..42048bceef84e --- /dev/null +++ b/tests/mir-opt/inline/forced_async.caller.ForceInline.diff @@ -0,0 +1,12 @@ +- // MIR for `caller` before ForceInline ++ // MIR for `caller` after ForceInline + + fn caller() -> {async fn body of caller()} { + let mut _0: {async fn body of caller()}; + + bb0: { + _0 = {coroutine@$DIR/forced_async.rs:10:19: 14:2 (#0)}; + return; + } + } + diff --git a/tests/mir-opt/inline/forced_async.rs b/tests/mir-opt/inline/forced_async.rs new file mode 100644 index 0000000000000..ed49eec01ee3b --- /dev/null +++ b/tests/mir-opt/inline/forced_async.rs @@ -0,0 +1,14 @@ +// EMIT_MIT_FOR_EACH_PANIC_STRATEGY +//@ compile-flags: -Copt-level=0 --crate-type=lib +//@ edition: 2021 +#![feature(rustc_attrs)] + +#[rustc_force_inline] +pub fn callee_forced() {} + +// EMIT_MIR forced_async.caller.ForceInline.diff +async fn caller() { + callee_forced(); + // CHECK-LABEL: fn caller( + // CHECK: (inlined callee_forced) +} diff --git a/tests/mir-opt/inline/forced_closure.caller-{closure#0}.ForceInline.diff b/tests/mir-opt/inline/forced_closure.caller-{closure#0}.ForceInline.diff new file mode 100644 index 0000000000000..8f50c20587d70 --- /dev/null +++ b/tests/mir-opt/inline/forced_closure.caller-{closure#0}.ForceInline.diff @@ -0,0 +1,21 @@ +- // MIR for `caller::{closure#0}` before ForceInline ++ // MIR for `caller::{closure#0}` after ForceInline + + fn caller::{closure#0}(_1: &{closure@$DIR/forced_closure.rs:10:6: 10:8}) -> () { + let mut _0: (); + let _2: (); ++ scope 1 (inlined callee_forced) { ++ } + + bb0: { + StorageLive(_2); +- _2 = callee_forced() -> [return: bb1, unwind continue]; +- } +- +- bb1: { + StorageDead(_2); + _0 = const (); + return; + } + } + diff --git a/tests/mir-opt/inline/forced_closure.rs b/tests/mir-opt/inline/forced_closure.rs new file mode 100644 index 0000000000000..ca5d4a8b7765d --- /dev/null +++ b/tests/mir-opt/inline/forced_closure.rs @@ -0,0 +1,15 @@ +// EMIT_MIT_FOR_EACH_PANIC_STRATEGY +//@ compile-flags: -Copt-level=0 --crate-type=lib +#![feature(rustc_attrs)] + +#[rustc_force_inline] +pub fn callee_forced() {} + +// EMIT_MIR forced_closure.caller-{closure#0}.ForceInline.diff +pub fn caller() { + (|| { + callee_forced(); + // CHECK-LABEL: fn caller::{closure#0}( + // CHECK: (inlined callee_forced) + })(); +} diff --git a/tests/ui/force-inlining/asm.rs b/tests/ui/force-inlining/asm.rs new file mode 100644 index 0000000000000..85014ffa51588 --- /dev/null +++ b/tests/ui/force-inlining/asm.rs @@ -0,0 +1,68 @@ +//@ build-fail +//@ compile-flags: --crate-type=lib --target thumbv4t-none-eabi +//@ needs-llvm-components: arm + +// Checks that forced inlining won't mix asm with incompatible instruction sets. + +#![crate_type = "lib"] +#![feature(rustc_attrs)] +#![feature(no_core, lang_items)] +#![no_core] + +#[lang = "sized"] +pub trait Sized {} +#[lang = "copy"] +pub trait Copy {} +#[lang = "freeze"] +pub unsafe trait Freeze {} + +#[lang = "start"] +fn start(_main: fn() -> T, _argc: isize, _argv: *const *const u8, _sigpipe: u8) -> isize { + 0 +} + +#[rustc_builtin_macro] +#[macro_export] +macro_rules! asm { + ("assembly template", + $(operands,)* + $(options($(option),*))? + ) => { + /* compiler built-in */ + }; +} + +#[instruction_set(arm::a32)] +#[rustc_force_inline] +fn instruction_set_a32() {} + +#[instruction_set(arm::t32)] +#[rustc_force_inline] +fn instruction_set_t32() {} + +#[rustc_force_inline] +fn instruction_set_default() {} + +#[rustc_force_inline] +fn inline_always_and_using_inline_asm() { + unsafe { asm!("/* do nothing */") }; +} + +#[instruction_set(arm::t32)] +pub fn t32() { + instruction_set_a32(); +//~^ ERROR `instruction_set_a32` could not be inlined into `t32` but is required to be inlined + instruction_set_t32(); + instruction_set_default(); + inline_always_and_using_inline_asm(); +//~^ ERROR `inline_always_and_using_inline_asm` could not be inlined into `t32` but is required to be inlined +} + +pub fn default() { + instruction_set_a32(); +//~^ ERROR `instruction_set_a32` could not be inlined into `default` but is required to be inlined + instruction_set_t32(); +//~^ ERROR `instruction_set_t32` could not be inlined into `default` but is required to be inlined + instruction_set_default(); + inline_always_and_using_inline_asm(); +} diff --git a/tests/ui/force-inlining/asm.stderr b/tests/ui/force-inlining/asm.stderr new file mode 100644 index 0000000000000..ef04688965bb3 --- /dev/null +++ b/tests/ui/force-inlining/asm.stderr @@ -0,0 +1,34 @@ +error: `instruction_set_a32` could not be inlined into `t32` but is required to be inlined + --> $DIR/asm.rs:53:5 + | +LL | instruction_set_a32(); + | ^^^^^^^^^^^^^^^^^^^^^ ...`instruction_set_a32` called here + | + = note: could not be inlined due to: incompatible instruction set + +error: `inline_always_and_using_inline_asm` could not be inlined into `t32` but is required to be inlined + --> $DIR/asm.rs:57:5 + | +LL | inline_always_and_using_inline_asm(); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ...`inline_always_and_using_inline_asm` called here + | + = note: could not be inlined due to: cannot move inline-asm across instruction sets + +error: `instruction_set_a32` could not be inlined into `default` but is required to be inlined + --> $DIR/asm.rs:62:5 + | +LL | instruction_set_a32(); + | ^^^^^^^^^^^^^^^^^^^^^ ...`instruction_set_a32` called here + | + = note: could not be inlined due to: incompatible instruction set + +error: `instruction_set_t32` could not be inlined into `default` but is required to be inlined + --> $DIR/asm.rs:64:5 + | +LL | instruction_set_t32(); + | ^^^^^^^^^^^^^^^^^^^^^ ...`instruction_set_t32` called here + | + = note: could not be inlined due to: incompatible instruction set + +error: aborting due to 4 previous errors + diff --git a/tests/ui/force-inlining/auxiliary/callees.rs b/tests/ui/force-inlining/auxiliary/callees.rs new file mode 100644 index 0000000000000..7468b8f10985d --- /dev/null +++ b/tests/ui/force-inlining/auxiliary/callees.rs @@ -0,0 +1,10 @@ +//@ compile-flags: --crate-type=lib +#![feature(rustc_attrs)] + +#[rustc_force_inline = "the test requires it"] +pub fn forced_with_reason() { +} + +#[rustc_force_inline] +pub fn forced() { +} diff --git a/tests/ui/force-inlining/cast.rs b/tests/ui/force-inlining/cast.rs new file mode 100644 index 0000000000000..8f51ec8397736 --- /dev/null +++ b/tests/ui/force-inlining/cast.rs @@ -0,0 +1,25 @@ +//@ check-fail +//@ compile-flags: --crate-type=lib +#![allow(internal_features)] +#![feature(rustc_attrs)] + +#[rustc_force_inline] +pub fn callee(x: isize) -> usize { unimplemented!() } + +fn a() { + let _: fn(isize) -> usize = callee; +//~^ ERROR cannot coerce functions which must be inlined to function pointers +} + +fn b() { + let _ = callee as fn(isize) -> usize; +//~^ ERROR non-primitive cast +} + +fn c() { + let _: [fn(isize) -> usize; 2] = [ + callee, +//~^ ERROR cannot coerce functions which must be inlined to function pointers + callee, + ]; +} diff --git a/tests/ui/force-inlining/cast.stderr b/tests/ui/force-inlining/cast.stderr new file mode 100644 index 0000000000000..116919e5fe7f0 --- /dev/null +++ b/tests/ui/force-inlining/cast.stderr @@ -0,0 +1,40 @@ +error[E0308]: cannot coerce functions which must be inlined to function pointers + --> $DIR/cast.rs:10:33 + | +LL | let _: fn(isize) -> usize = callee; + | ------------------ ^^^^^^ cannot coerce functions which must be inlined to function pointers + | | + | expected due to this + | + = note: expected fn pointer `fn(_) -> _` + found fn item `fn(_) -> _ {callee}` + = note: fn items are distinct from fn pointers +help: consider casting to a fn pointer + | +LL | let _: fn(isize) -> usize = callee as fn(isize) -> usize; + | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +error[E0605]: non-primitive cast: `fn(isize) -> usize {callee}` as `fn(isize) -> usize` + --> $DIR/cast.rs:15:13 + | +LL | let _ = callee as fn(isize) -> usize; + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ invalid cast + +error[E0308]: cannot coerce functions which must be inlined to function pointers + --> $DIR/cast.rs:21:9 + | +LL | callee, + | ^^^^^^ cannot coerce functions which must be inlined to function pointers + | + = note: expected fn pointer `fn(_) -> _` + found fn item `fn(_) -> _ {callee}` + = note: fn items are distinct from fn pointers +help: consider casting to a fn pointer + | +LL | callee as fn(isize) -> usize, + | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +error: aborting due to 3 previous errors + +Some errors have detailed explanations: E0308, E0605. +For more information about an error, try `rustc --explain E0308`. diff --git a/tests/ui/force-inlining/cross-crate.rs b/tests/ui/force-inlining/cross-crate.rs new file mode 100644 index 0000000000000..a3f0c2c873ea8 --- /dev/null +++ b/tests/ui/force-inlining/cross-crate.rs @@ -0,0 +1,13 @@ +//@ aux-build:callees.rs +//@ build-pass +//@ compile-flags: --crate-type=lib + +extern crate callees; + +// Test that forced inlining across crates works as expected. + +pub fn caller() { + callees::forced(); + + callees::forced_with_reason(); +} diff --git a/tests/ui/force-inlining/deny-async.rs b/tests/ui/force-inlining/deny-async.rs new file mode 100644 index 0000000000000..d5b6dc000f70d --- /dev/null +++ b/tests/ui/force-inlining/deny-async.rs @@ -0,0 +1,25 @@ +//@ check-fail +//@ compile-flags: --crate-type=lib +//@ edition: 2021 +#![allow(internal_features)] +#![feature(rustc_attrs)] + +// Test that forced inlining into async functions w/ errors works as expected. + +#[rustc_no_mir_inline] +#[rustc_force_inline] +pub fn callee() { +} + +#[rustc_no_mir_inline] +#[rustc_force_inline = "the test requires it"] +pub fn callee_justified() { +} + +async fn async_caller() { + callee(); +//~^ ERROR `callee` could not be inlined into `async_caller::{closure#0}` but is required to be inlined + + callee_justified(); +//~^ ERROR `callee_justified` could not be inlined into `async_caller::{closure#0}` but is required to be inlined +} diff --git a/tests/ui/force-inlining/deny-async.stderr b/tests/ui/force-inlining/deny-async.stderr new file mode 100644 index 0000000000000..7b077174ee89e --- /dev/null +++ b/tests/ui/force-inlining/deny-async.stderr @@ -0,0 +1,19 @@ +error: `callee` could not be inlined into `async_caller::{closure#0}` but is required to be inlined + --> $DIR/deny-async.rs:20:5 + | +LL | callee(); + | ^^^^^^^^ ...`callee` called here + | + = note: could not be inlined due to: #[rustc_no_mir_inline] + +error: `callee_justified` could not be inlined into `async_caller::{closure#0}` but is required to be inlined + --> $DIR/deny-async.rs:23:5 + | +LL | callee_justified(); + | ^^^^^^^^^^^^^^^^^^ ...`callee_justified` called here + | + = note: could not be inlined due to: #[rustc_no_mir_inline] + = note: `callee_justified` is required to be inlined to: the test requires it + +error: aborting due to 2 previous errors + diff --git a/tests/ui/force-inlining/deny-closure.rs b/tests/ui/force-inlining/deny-closure.rs new file mode 100644 index 0000000000000..0057f1e78540c --- /dev/null +++ b/tests/ui/force-inlining/deny-closure.rs @@ -0,0 +1,26 @@ +//@ build-fail +//@ compile-flags: --crate-type=lib +#![allow(internal_features)] +#![feature(rustc_attrs)] + +// Test that forced inlining into closures w/ errors works as expected. + +#[rustc_no_mir_inline] +#[rustc_force_inline] +pub fn callee() { +} + +#[rustc_no_mir_inline] +#[rustc_force_inline = "the test requires it"] +pub fn callee_justified() { +} + +pub fn caller() { + (|| { + callee(); +//~^ ERROR `callee` could not be inlined into `caller::{closure#0}` but is required to be inlined + + callee_justified(); +//~^ ERROR `callee_justified` could not be inlined into `caller::{closure#0}` but is required to be inlined + })(); +} diff --git a/tests/ui/force-inlining/deny-closure.stderr b/tests/ui/force-inlining/deny-closure.stderr new file mode 100644 index 0000000000000..9600fa68f540a --- /dev/null +++ b/tests/ui/force-inlining/deny-closure.stderr @@ -0,0 +1,30 @@ +error: `callee` could not be inlined into `caller::{closure#0}` but is required to be inlined + --> $DIR/deny-closure.rs:20:9 + | +LL | callee(); + | ^^^^^^^^ ...`callee` called here + | + = note: could not be inlined due to: #[rustc_no_mir_inline] + +error: `callee_justified` could not be inlined into `caller::{closure#0}` but is required to be inlined + --> $DIR/deny-closure.rs:23:9 + | +LL | callee_justified(); + | ^^^^^^^^^^^^^^^^^^ ...`callee_justified` called here + | + = note: could not be inlined due to: #[rustc_no_mir_inline] + = note: `callee_justified` is required to be inlined to: the test requires it + +note: the above error was encountered while instantiating `fn caller::{closure#0}` + --> $DIR/deny-closure.rs:19:5 + | +LL | / (|| { +LL | | callee(); +LL | | +... | +LL | | +LL | | })(); + | |________^ + +error: aborting due to 2 previous errors + diff --git a/tests/ui/force-inlining/deny.rs b/tests/ui/force-inlining/deny.rs new file mode 100644 index 0000000000000..a19e820cff28d --- /dev/null +++ b/tests/ui/force-inlining/deny.rs @@ -0,0 +1,24 @@ +//@ build-fail +//@ compile-flags: --crate-type=lib +#![allow(internal_features)] +#![feature(rustc_attrs)] + +// Test that forced inlining w/ errors works as expected. + +#[rustc_no_mir_inline] +#[rustc_force_inline] +pub fn callee() { +} + +#[rustc_no_mir_inline] +#[rustc_force_inline = "the test requires it"] +pub fn callee_justified() { +} + +pub fn caller() { + callee(); +//~^ ERROR `callee` could not be inlined into `caller` but is required to be inlined + + callee_justified(); +//~^ ERROR `callee_justified` could not be inlined into `caller` but is required to be inlined +} diff --git a/tests/ui/force-inlining/deny.stderr b/tests/ui/force-inlining/deny.stderr new file mode 100644 index 0000000000000..3983bead182eb --- /dev/null +++ b/tests/ui/force-inlining/deny.stderr @@ -0,0 +1,19 @@ +error: `callee` could not be inlined into `caller` but is required to be inlined + --> $DIR/deny.rs:19:5 + | +LL | callee(); + | ^^^^^^^^ ...`callee` called here + | + = note: could not be inlined due to: #[rustc_no_mir_inline] + +error: `callee_justified` could not be inlined into `caller` but is required to be inlined + --> $DIR/deny.rs:22:5 + | +LL | callee_justified(); + | ^^^^^^^^^^^^^^^^^^ ...`callee_justified` called here + | + = note: could not be inlined due to: #[rustc_no_mir_inline] + = note: `callee_justified` is required to be inlined to: the test requires it + +error: aborting due to 2 previous errors + diff --git a/tests/ui/force-inlining/gate.rs b/tests/ui/force-inlining/gate.rs new file mode 100644 index 0000000000000..d6a01a74a44b5 --- /dev/null +++ b/tests/ui/force-inlining/gate.rs @@ -0,0 +1,12 @@ +//@ compile-flags: --crate-type=lib +#![allow(internal_features)] + +#[rustc_force_inline] +//~^ ERROR #![rustc_force_inline] forces a free function to be inlined +pub fn bare() { +} + +#[rustc_force_inline = "the test requires it"] +//~^ ERROR #![rustc_force_inline] forces a free function to be inlined +pub fn justified() { +} diff --git a/tests/ui/force-inlining/gate.stderr b/tests/ui/force-inlining/gate.stderr new file mode 100644 index 0000000000000..e3973a08c232b --- /dev/null +++ b/tests/ui/force-inlining/gate.stderr @@ -0,0 +1,21 @@ +error[E0658]: #![rustc_force_inline] forces a free function to be inlined + --> $DIR/gate.rs:4:1 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ + | + = help: add `#![feature(rustc_attrs)]` to the crate attributes to enable + = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date + +error[E0658]: #![rustc_force_inline] forces a free function to be inlined + --> $DIR/gate.rs:9:1 + | +LL | #[rustc_force_inline = "the test requires it"] + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | + = help: add `#![feature(rustc_attrs)]` to the crate attributes to enable + = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date + +error: aborting due to 2 previous errors + +For more information about this error, try `rustc --explain E0658`. diff --git a/tests/ui/force-inlining/invalid.rs b/tests/ui/force-inlining/invalid.rs new file mode 100644 index 0000000000000..7574078b245c2 --- /dev/null +++ b/tests/ui/force-inlining/invalid.rs @@ -0,0 +1,164 @@ +//@ edition: 2024 +#![allow(internal_features, unused_imports, unused_macros)] +#![feature(extern_types)] +#![feature(gen_blocks)] +#![feature(rustc_attrs)] +#![feature(stmt_expr_attributes)] +#![feature(trait_alias)] + +// Test that invalid force inlining attributes error as expected. + +#[rustc_force_inline("foo")] +//~^ ERROR malformed `rustc_force_inline` attribute input +pub fn forced1() { +} + +#[rustc_force_inline(bar, baz)] +//~^ ERROR malformed `rustc_force_inline` attribute input +pub fn forced2() { +} + +#[rustc_force_inline(2)] +//~^ ERROR malformed `rustc_force_inline` attribute input +pub fn forced3() { +} + +#[rustc_force_inline = 2] +//~^ ERROR malformed `rustc_force_inline` attribute input +pub fn forced4() { +} + +#[rustc_force_inline] +//~^ ERROR attribute should be applied to a function +extern crate std as other_std; + +#[rustc_force_inline] +//~^ ERROR attribute should be applied to a function +use std::collections::HashMap; + +#[rustc_force_inline] +//~^ ERROR attribute should be applied to a function +static _FOO: &'static str = "FOO"; + +#[rustc_force_inline] +//~^ ERROR attribute should be applied to a function +const _BAR: u32 = 3; + +#[rustc_force_inline] +//~^ ERROR attribute should be applied to a function +mod foo { } + +#[rustc_force_inline] +//~^ ERROR attribute should be applied to a function +unsafe extern "C" { + #[rustc_force_inline] +//~^ ERROR attribute should be applied to a function + static X: &'static u32; + + #[rustc_force_inline] +//~^ ERROR attribute should be applied to a function + type Y; + + #[rustc_force_inline] +//~^ ERROR attribute should be applied to a function + fn foo(); +} + +#[rustc_force_inline] +//~^ ERROR attribute should be applied to a function +type Foo = u32; + +#[rustc_force_inline] +//~^ ERROR attribute should be applied to a function +enum Bar<#[rustc_force_inline] T> { +//~^ ERROR attribute should be applied to a function + #[rustc_force_inline] +//~^ ERROR attribute should be applied to a function + Baz(std::marker::PhantomData), +} + +#[rustc_force_inline] +//~^ ERROR attribute should be applied to a function +struct Qux { + #[rustc_force_inline] +//~^ ERROR attribute should be applied to a function + field: u32, +} + +#[rustc_force_inline] +//~^ ERROR attribute should be applied to a function +union FooBar { + x: u32, + y: u32, +} + +#[rustc_force_inline] +//~^ ERROR attribute should be applied to a function +trait FooBaz { + #[rustc_force_inline] +//~^ ERROR attribute should be applied to a function + type Foo; + #[rustc_force_inline] +//~^ ERROR attribute should be applied to a function + const Bar: i32; + + #[rustc_force_inline] +//~^ ERROR attribute should be applied to a function + fn foo() {} +} + +#[rustc_force_inline] +//~^ ERROR attribute should be applied to a function +trait FooQux = FooBaz; + +#[rustc_force_inline] +//~^ ERROR attribute should be applied to a function +impl Bar { + #[rustc_force_inline] +//~^ ERROR attribute should be applied to a function + fn foo() {} +} + +#[rustc_force_inline] +//~^ ERROR attribute should be applied to a function +impl FooBaz for Bar { + type Foo = u32; + const Bar: i32 = 3; +} + +#[rustc_force_inline] +//~^ ERROR attribute should be applied to a function +macro_rules! barqux { ($foo:tt) => { $foo }; } + +fn barqux(#[rustc_force_inline] _x: u32) {} +//~^ ERROR allow, cfg, cfg_attr, deny, expect, forbid, and warn are the only allowed built-in attributes in function parameters +//~^^ ERROR attribute should be applied to a function + +#[rustc_force_inline] +//~^ ERROR attribute cannot be applied to a `async`, `gen` or `async gen` function +async fn async_foo() {} + +#[rustc_force_inline] +//~^ ERROR attribute cannot be applied to a `async`, `gen` or `async gen` function +gen fn gen_foo() {} + +#[rustc_force_inline] +//~^ ERROR attribute cannot be applied to a `async`, `gen` or `async gen` function +async gen fn async_gen_foo() {} + +fn main() { + let _x = #[rustc_force_inline] || { }; +//~^ ERROR attribute should be applied to a function + let _y = #[rustc_force_inline] 3 + 4; +//~^ ERROR attribute should be applied to a function + #[rustc_force_inline] +//~^ ERROR attribute should be applied to a function + let _z = 3; + + match _z { + #[rustc_force_inline] +//~^ ERROR attribute should be applied to a function + 1 => (), + _ => (), + } +} diff --git a/tests/ui/force-inlining/invalid.stderr b/tests/ui/force-inlining/invalid.stderr new file mode 100644 index 0000000000000..36d824fd05fc7 --- /dev/null +++ b/tests/ui/force-inlining/invalid.stderr @@ -0,0 +1,378 @@ +error: malformed `rustc_force_inline` attribute input + --> $DIR/invalid.rs:11:1 + | +LL | #[rustc_force_inline("foo")] + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | +help: the following are the possible correct uses + | +LL | #[rustc_force_inline = "reason"] + | +LL | #[rustc_force_inline] + | + +error: malformed `rustc_force_inline` attribute input + --> $DIR/invalid.rs:16:1 + | +LL | #[rustc_force_inline(bar, baz)] + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | +help: the following are the possible correct uses + | +LL | #[rustc_force_inline = "reason"] + | +LL | #[rustc_force_inline] + | + +error: malformed `rustc_force_inline` attribute input + --> $DIR/invalid.rs:21:1 + | +LL | #[rustc_force_inline(2)] + | ^^^^^^^^^^^^^^^^^^^^^^^^ + | +help: the following are the possible correct uses + | +LL | #[rustc_force_inline = "reason"] + | +LL | #[rustc_force_inline] + | + +error: malformed `rustc_force_inline` attribute input + --> $DIR/invalid.rs:26:1 + | +LL | #[rustc_force_inline = 2] + | ^^^^^^^^^^^^^^^^^^^^^^^^^ + | +help: the following are the possible correct uses + | +LL | #[rustc_force_inline = "reason"] + | +LL | #[rustc_force_inline] + | + +error: allow, cfg, cfg_attr, deny, expect, forbid, and warn are the only allowed built-in attributes in function parameters + --> $DIR/invalid.rs:133:11 + | +LL | fn barqux(#[rustc_force_inline] _x: u32) {} + | ^^^^^^^^^^^^^^^^^^^^^ + +error: attribute should be applied to a function + --> $DIR/invalid.rs:31:1 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | extern crate std as other_std; + | ------------------------------ not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:35:1 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | use std::collections::HashMap; + | ------------------------------ not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:39:1 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | static _FOO: &'static str = "FOO"; + | ---------------------------------- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:43:1 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | const _BAR: u32 = 3; + | -------------------- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:47:1 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | mod foo { } + | ----------- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:51:1 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | / unsafe extern "C" { +LL | | #[rustc_force_inline] +LL | | +LL | | static X: &'static u32; +... | +LL | | fn foo(); +LL | | } + | |_- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:67:1 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | type Foo = u32; + | --------------- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:71:1 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | / enum Bar<#[rustc_force_inline] T> { +LL | | +LL | | #[rustc_force_inline] +LL | | +LL | | Baz(std::marker::PhantomData), +LL | | } + | |_- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:73:10 + | +LL | enum Bar<#[rustc_force_inline] T> { + | ^^^^^^^^^^^^^^^^^^^^^ - not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:75:5 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | Baz(std::marker::PhantomData), + | -------------------------------- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:80:1 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | / struct Qux { +LL | | #[rustc_force_inline] +LL | | +LL | | field: u32, +LL | | } + | |_- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:83:5 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | field: u32, + | ---------- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:88:1 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | / union FooBar { +LL | | x: u32, +LL | | y: u32, +LL | | } + | |_- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:95:1 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | / trait FooBaz { +LL | | #[rustc_force_inline] +LL | | +LL | | type Foo; +... | +LL | | fn foo() {} +LL | | } + | |_- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:110:1 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | trait FooQux = FooBaz; + | ---------------------- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:114:1 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | / impl Bar { +LL | | #[rustc_force_inline] +LL | | +LL | | fn foo() {} +LL | | } + | |_- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:122:1 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | / impl FooBaz for Bar { +LL | | type Foo = u32; +LL | | const Bar: i32 = 3; +LL | | } + | |_- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:129:1 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | macro_rules! barqux { ($foo:tt) => { $foo }; } + | ---------------------------------------------- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:133:11 + | +LL | fn barqux(#[rustc_force_inline] _x: u32) {} + | ^^^^^^^^^^^^^^^^^^^^^-------- + | | + | not a function definition + +error: attribute cannot be applied to a `async`, `gen` or `async gen` function + --> $DIR/invalid.rs:137:1 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | async fn async_foo() {} + | -------------------- `async`, `gen` or `async gen` function + +error: attribute cannot be applied to a `async`, `gen` or `async gen` function + --> $DIR/invalid.rs:141:1 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | gen fn gen_foo() {} + | ---------------- `async`, `gen` or `async gen` function + +error: attribute cannot be applied to a `async`, `gen` or `async gen` function + --> $DIR/invalid.rs:145:1 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | async gen fn async_gen_foo() {} + | ---------------------------- `async`, `gen` or `async gen` function + +error: attribute should be applied to a function + --> $DIR/invalid.rs:150:14 + | +LL | let _x = #[rustc_force_inline] || { }; + | ^^^^^^^^^^^^^^^^^^^^^ ------ not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:152:14 + | +LL | let _y = #[rustc_force_inline] 3 + 4; + | ^^^^^^^^^^^^^^^^^^^^^ - not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:154:5 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | let _z = 3; + | ----------- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:159:9 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | 1 => (), + | ------- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:98:5 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | type Foo; + | --------- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:101:5 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | const Bar: i32; + | --------------- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:105:5 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | fn foo() {} + | ----------- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:117:5 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | fn foo() {} + | ----------- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:54:5 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | static X: &'static u32; + | ----------------------- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:58:5 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | type Y; + | ------- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:62:5 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | fn foo(); + | --------- not a function definition + +error: aborting due to 38 previous errors + diff --git a/tests/ui/force-inlining/shims.rs b/tests/ui/force-inlining/shims.rs new file mode 100644 index 0000000000000..03b7d07cda39e --- /dev/null +++ b/tests/ui/force-inlining/shims.rs @@ -0,0 +1,9 @@ +//@ build-pass +#![allow(internal_features)] +#![feature(rustc_attrs)] + +#[rustc_force_inline] +fn f() {} +fn g(t: T) { t(); } + +fn main() { g(f); }