diff --git a/cranelift/codegen/src/ir/layout.rs b/cranelift/codegen/src/ir/layout.rs index 37842b59bef1..4aeaad9a094f 100644 --- a/cranelift/codegen/src/ir/layout.rs +++ b/cranelift/codegen/src/ir/layout.rs @@ -469,6 +469,17 @@ impl Layout { pub fn next_block(&self, block: Block) -> Option { self.blocks[block].next.expand() } + + /// Mark a block as "cold". This will try to move it out of the + /// ordinary path of execution when lowered to machine code. + pub fn mark_cold_block(&mut self, block: Block) { + self.blocks[block].cold = true; + } + + /// Is the given block cold? + pub fn is_cold(&self, block: Block) -> bool { + self.blocks[block].cold + } } #[derive(Clone, Debug, Default)] @@ -478,6 +489,7 @@ struct BlockNode { first_inst: PackedOption, last_inst: PackedOption, seq: SequenceNumber, + cold: bool, } /// Iterate over blocks in layout order. See [crate::ir::layout::Layout::blocks]. diff --git a/cranelift/codegen/src/machinst/blockorder.rs b/cranelift/codegen/src/machinst/blockorder.rs index 73c5ea24539c..3cdf1228ca1f 100644 --- a/cranelift/codegen/src/machinst/blockorder.rs +++ b/cranelift/codegen/src/machinst/blockorder.rs @@ -378,11 +378,24 @@ impl BlockLoweringOrder { postorder.reverse(); let mut rpo = postorder; + + // Step 3: sink any cold blocks to the end of the + // function. Put the "deferred last" block truly at the end; + // this is a correctness requirement (for fallthrough + // returns). + rpo.sort_by_key(|block| { + block + .0 + .orig_block() + .map(|block| f.layout.is_cold(block)) + .unwrap_or(false) + }); + if let Some(d) = deferred_last { rpo.push(d); } - // Step 3: now that we have RPO, build the BlockIndex/BB fwd/rev maps. + // Step 4: now that we have RPO, build the BlockIndex/BB fwd/rev maps. let mut lowered_order = vec![]; let mut lowered_succ_ranges = vec![]; let mut lb_to_bindex = FxHashMap::default(); diff --git a/cranelift/frontend/src/frontend.rs b/cranelift/frontend/src/frontend.rs index 7611e69c0a85..ba3d4b9689f4 100644 --- a/cranelift/frontend/src/frontend.rs +++ b/cranelift/frontend/src/frontend.rs @@ -229,6 +229,11 @@ impl<'a> FunctionBuilder<'a> { block } + /// Mark a block as cold. + pub fn mark_cold_block(&mut self, block: Block) { + self.func.layout.mark_cold_block(block); + } + /// Insert `block` in the layout *after* the existing block `after`. pub fn insert_block_after(&mut self, block: Block, after: Block) { self.func.layout.insert_block_after(block, after);