Auto merge of rust-lang#95867 - cjgillot:fixed-size, r=oli-obk

Skip `Lazy` for some metadata tables

Some metadata tables encode their entries indirectly, through the Lazy construct. This is useful when dealing with variable length encoding, but incurs the extra cost of one u32.

Meanwhile, some fields can be encoded in a single u8, or can use a short fixed-length encoding. This PR proposes to do so, and avoid the overhead.

compiler/rustc_metadata/src/lib.rs

@@ -5,7 +5,9 @@
 #![feature(nll)]
 #![feature(once_cell)]
 #![feature(proc_macro_internals)]
+#![feature(macro_metavar_expr)]
 #![feature(min_specialization)]
+#![feature(slice_as_chunks)]
 #![feature(try_blocks)]
 #![feature(never_type)]
 #![recursion_limit = "256"]

compiler/rustc_metadata/src/rmeta/decoder.rs

@@ -292,6 +292,18 @@ trait LazyQueryDecodable<'a, 'tcx, T> {
     ) -> T;
 }
 
+impl<'a, 'tcx, T> LazyQueryDecodable<'a, 'tcx, T> for T {
+    fn decode_query(self, _: CrateMetadataRef<'a>, _: TyCtxt<'tcx>, _: impl FnOnce() -> !) -> T {
+        self
+    }
+}
+
+impl<'a, 'tcx, T> LazyQueryDecodable<'a, 'tcx, T> for Option<T> {
+    fn decode_query(self, _: CrateMetadataRef<'a>, _: TyCtxt<'tcx>, err: impl FnOnce() -> !) -> T {
+        if let Some(l) = self { l } else { err() }
+    }
+}
+
 impl<'a, 'tcx, T> LazyQueryDecodable<'a, 'tcx, T> for Option<Lazy<T>>
 where
     T: Decodable<DecodeContext<'a, 'tcx>>,
@@ -376,6 +388,17 @@ impl<'a, 'tcx> LazyQueryDecodable<'a, 'tcx, Option<DeprecationEntry>>
     }
 }
 
+impl<'a, 'tcx> LazyQueryDecodable<'a, 'tcx, Option<DefId>> for Option<RawDefId> {
+    fn decode_query(
+        self,
+        cdata: CrateMetadataRef<'a>,
+        _: TyCtxt<'tcx>,
+        _: impl FnOnce() -> !,
+    ) -> Option<DefId> {
+        self.map(|raw_def_id| raw_def_id.decode(cdata))
+    }
+}
+
 impl<'a, 'tcx> DecodeContext<'a, 'tcx> {
     #[inline]
     fn tcx(&self) -> TyCtxt<'tcx> {
@@ -394,8 +417,9 @@ impl<'a, 'tcx> DecodeContext<'a, 'tcx> {
         self.cdata.unwrap()
     }
 
+    #[inline]
     fn map_encoded_cnum_to_current(&self, cnum: CrateNum) -> CrateNum {
-        if cnum == LOCAL_CRATE { self.cdata().cnum } else { self.cdata().cnum_map[cnum] }
+        self.cdata().map_encoded_cnum_to_current(cnum)
     }
 
     fn read_lazy_with_meta<T: ?Sized + LazyMeta>(&mut self, meta: T::Meta) -> Lazy<T> {
@@ -706,8 +730,7 @@ impl<'a, 'tcx, T: Decodable<DecodeContext<'a, 'tcx>>> Decodable<DecodeContext<'a
     }
 }
 
-impl<'a, 'tcx, I: Idx, T: Decodable<DecodeContext<'a, 'tcx>>> Decodable<DecodeContext<'a, 'tcx>>
-    for Lazy<Table<I, T>>
+impl<'a, 'tcx, I: Idx, T> Decodable<DecodeContext<'a, 'tcx>> for Lazy<Table<I, T>>
 where
     Option<T>: FixedSizeEncoding,
 {
@@ -844,6 +867,11 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
         self.root.tables.kind.get(self, item_id).map(|k| k.decode(self))
     }
 
+    #[inline]
+    pub(super) fn map_encoded_cnum_to_current(self, cnum: CrateNum) -> CrateNum {
+        if cnum == LOCAL_CRATE { self.cnum } else { self.cnum_map[cnum] }
+    }
+
     fn kind(self, item_id: DefIndex) -> EntryKind {
         self.maybe_kind(item_id).unwrap_or_else(|| {
             bug!(
@@ -856,16 +884,14 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
     }
 
     fn def_kind(self, item_id: DefIndex) -> DefKind {
-        self.root.tables.opt_def_kind.get(self, item_id).map(|k| k.decode(self)).unwrap_or_else(
-            || {
-                bug!(
-                    "CrateMetadata::def_kind({:?}): id not found, in crate {:?} with number {}",
-                    item_id,
-                    self.root.name,
-                    self.cnum,
-                )
-            },
-        )
+        self.root.tables.opt_def_kind.get(self, item_id).unwrap_or_else(|| {
+            bug!(
+                "CrateMetadata::def_kind({:?}): id not found, in crate {:?} with number {}",
+                item_id,
+                self.root.name,
+                self.cnum,
+            )
+        })
     }
 
     fn get_span(self, index: DefIndex, sess: &Session) -> Span {
@@ -1449,9 +1475,9 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
         index: DefIndex,
         def_path_hashes: &mut FxHashMap<DefIndex, DefPathHash>,
     ) -> DefPathHash {
-        *def_path_hashes.entry(index).or_insert_with(|| {
-            self.root.tables.def_path_hashes.get(self, index).unwrap().decode(self)
-        })
+        *def_path_hashes
+            .entry(index)
+            .or_insert_with(|| self.root.tables.def_path_hashes.get(self, index).unwrap())
     }
 
     #[inline]

compiler/rustc_metadata/src/rmeta/encoder.rs

@@ -147,8 +147,7 @@ impl<'a, 'tcx, T: Encodable<EncodeContext<'a, 'tcx>>> Encodable<EncodeContext<'a
     }
 }
 
-impl<'a, 'tcx, I: Idx, T: Encodable<EncodeContext<'a, 'tcx>>> Encodable<EncodeContext<'a, 'tcx>>
-    for Lazy<Table<I, T>>
+impl<'a, 'tcx, I: Idx, T> Encodable<EncodeContext<'a, 'tcx>> for Lazy<Table<I, T>>
 where
     Option<T>: FixedSizeEncoding,
 {
@@ -461,16 +460,15 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
                 .chain(self.tcx.resolutions(()).proc_macros.iter().map(|p| p.local_def_index))
             {
                 let def_key = self.lazy(table.def_key(def_index));
-                let def_path_hash = self.lazy(table.def_path_hash(def_index));
+                let def_path_hash = table.def_path_hash(def_index);
                 self.tables.def_keys.set(def_index, def_key);
                 self.tables.def_path_hashes.set(def_index, def_path_hash);
             }
         } else {
             for (def_index, def_key, def_path_hash) in table.enumerated_keys_and_path_hashes() {
                 let def_key = self.lazy(def_key);
-                let def_path_hash = self.lazy(def_path_hash);
                 self.tables.def_keys.set(def_index, def_key);
-                self.tables.def_path_hashes.set(def_index, def_path_hash);
+                self.tables.def_path_hashes.set(def_index, *def_path_hash);
             }
         }
     }
@@ -988,7 +986,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
             let def_id = local_id.to_def_id();
             let def_kind = tcx.opt_def_kind(local_id);
             let Some(def_kind) = def_kind else { continue };
-            record!(self.tables.opt_def_kind[def_id] <- def_kind);
+            self.tables.opt_def_kind.set(def_id.index, def_kind);
             record!(self.tables.def_span[def_id] <- tcx.def_span(def_id));
             record!(self.tables.attributes[def_id] <- tcx.get_attrs(def_id));
             record!(self.tables.expn_that_defined[def_id] <- self.tcx.expn_that_defined(def_id));
@@ -1048,7 +1046,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
         };
 
         record!(self.tables.kind[def_id] <- EntryKind::Variant(self.lazy(data)));
-        record!(self.tables.impl_constness[def_id] <- hir::Constness::Const);
+        self.tables.impl_constness.set(def_id.index, hir::Constness::Const);
         record!(self.tables.children[def_id] <- variant.fields.iter().map(|f| {
             assert!(f.did.is_local());
             f.did.index
@@ -1078,7 +1076,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
         };
 
         record!(self.tables.kind[def_id] <- EntryKind::Variant(self.lazy(data)));
-        record!(self.tables.impl_constness[def_id] <- hir::Constness::Const);
+        self.tables.impl_constness.set(def_id.index, hir::Constness::Const);
         self.encode_item_type(def_id);
         if variant.ctor_kind == CtorKind::Fn {
             record!(self.tables.fn_sig[def_id] <- tcx.fn_sig(def_id));
@@ -1157,7 +1155,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
         };
 
         record!(self.tables.repr_options[def_id] <- adt_def.repr());
-        record!(self.tables.impl_constness[def_id] <- hir::Constness::Const);
+        self.tables.impl_constness.set(def_id.index, hir::Constness::Const);
         record!(self.tables.kind[def_id] <- EntryKind::Struct(self.lazy(data)));
         self.encode_item_type(def_id);
         if variant.ctor_kind == CtorKind::Fn {
@@ -1207,8 +1205,8 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
                         record!(self.tables.fn_arg_names[def_id] <- self.tcx.hir().body_param_names(body))
                     }
                 };
-                record!(self.tables.asyncness[def_id] <- m_sig.header.asyncness);
-                record!(self.tables.impl_constness[def_id] <- hir::Constness::NotConst);
+                self.tables.asyncness.set(def_id.index, m_sig.header.asyncness);
+                self.tables.impl_constness.set(def_id.index, hir::Constness::NotConst);
                 record!(self.tables.kind[def_id] <- EntryKind::AssocFn(self.lazy(AssocFnData {
                     container,
                     has_self: trait_item.fn_has_self_parameter,
@@ -1265,15 +1263,15 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
             }
             ty::AssocKind::Fn => {
                 let hir::ImplItemKind::Fn(ref sig, body) = ast_item.kind else { bug!() };
-                record!(self.tables.asyncness[def_id] <- sig.header.asyncness);
+                self.tables.asyncness.set(def_id.index, sig.header.asyncness);
                 record!(self.tables.fn_arg_names[def_id] <- self.tcx.hir().body_param_names(body));
                 // Can be inside `impl const Trait`, so using sig.header.constness is not reliable
                 let constness = if self.tcx.is_const_fn_raw(def_id) {
                     hir::Constness::Const
                 } else {
                     hir::Constness::NotConst
                 };
-                record!(self.tables.impl_constness[def_id] <- constness);
+                self.tables.impl_constness.set(def_id.index, constness);
                 record!(self.tables.kind[def_id] <- EntryKind::AssocFn(self.lazy(AssocFnData {
                     container,
                     has_self: impl_item.fn_has_self_parameter,
@@ -1286,7 +1284,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
         self.encode_ident_span(def_id, impl_item.ident(self.tcx));
         self.encode_item_type(def_id);
         if let Some(trait_item_def_id) = impl_item.trait_item_def_id {
-            record!(self.tables.trait_item_def_id[def_id] <- trait_item_def_id);
+            self.tables.trait_item_def_id.set(def_id.index, trait_item_def_id.into());
         }
         if impl_item.kind == ty::AssocKind::Fn {
             record!(self.tables.fn_sig[def_id] <- tcx.fn_sig(def_id));
@@ -1394,9 +1392,9 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
                 EntryKind::Const
             }
             hir::ItemKind::Fn(ref sig, .., body) => {
-                record!(self.tables.asyncness[def_id] <- sig.header.asyncness);
+                self.tables.asyncness.set(def_id.index, sig.header.asyncness);
                 record!(self.tables.fn_arg_names[def_id] <- self.tcx.hir().body_param_names(body));
-                record!(self.tables.impl_constness[def_id] <- sig.header.constness);
+                self.tables.impl_constness.set(def_id.index, sig.header.constness);
                 EntryKind::Fn
             }
             hir::ItemKind::Macro(ref macro_def, _) => {
@@ -1420,7 +1418,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
             hir::ItemKind::Struct(ref struct_def, _) => {
                 let adt_def = self.tcx.adt_def(def_id);
                 record!(self.tables.repr_options[def_id] <- adt_def.repr());
-                record!(self.tables.impl_constness[def_id] <- hir::Constness::Const);
+                self.tables.impl_constness.set(def_id.index, hir::Constness::Const);
 
                 // Encode def_ids for each field and method
                 // for methods, write all the stuff get_trait_method
@@ -1450,15 +1448,15 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
                 }))
             }
             hir::ItemKind::Impl(hir::Impl { defaultness, constness, .. }) => {
-                record!(self.tables.impl_defaultness[def_id] <- defaultness);
-                record!(self.tables.impl_constness[def_id] <- constness);
+                self.tables.impl_defaultness.set(def_id.index, defaultness);
+                self.tables.impl_constness.set(def_id.index, constness);
 
                 let trait_ref = self.tcx.impl_trait_ref(def_id);
                 if let Some(trait_ref) = trait_ref {
                     let trait_def = self.tcx.trait_def(trait_ref.def_id);
                     if let Some(mut an) = trait_def.ancestors(self.tcx, def_id).ok() {
                         if let Some(specialization_graph::Node::Impl(parent)) = an.nth(1) {
-                            record!(self.tables.impl_parent[def_id] <- parent);
+                            self.tables.impl_parent.set(def_id.index, parent.into());
                         }
                     }
 
@@ -1472,7 +1470,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
                 }
 
                 let polarity = self.tcx.impl_polarity(def_id);
-                record!(self.tables.impl_polarity[def_id] <- polarity);
+                self.tables.impl_polarity.set(def_id.index, polarity);
 
                 EntryKind::Impl
             }
@@ -1644,7 +1642,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
                 self.tables.proc_macro_quoted_spans.set(i, span);
             }
 
-            record!(self.tables.opt_def_kind[LOCAL_CRATE.as_def_id()] <- DefKind::Mod);
+            self.tables.opt_def_kind.set(LOCAL_CRATE.as_def_id().index, DefKind::Mod);
             record!(self.tables.def_span[LOCAL_CRATE.as_def_id()] <- tcx.def_span(LOCAL_CRATE.as_def_id()));
             record!(self.tables.attributes[LOCAL_CRATE.as_def_id()] <- tcx.get_attrs(LOCAL_CRATE.as_def_id()));
             record!(self.tables.visibility[LOCAL_CRATE.as_def_id()] <- tcx.visibility(LOCAL_CRATE.as_def_id()));
@@ -1685,7 +1683,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
                 def_key.disambiguated_data.data = DefPathData::MacroNs(name);
 
                 let def_id = id.to_def_id();
-                record!(self.tables.opt_def_kind[def_id] <- DefKind::Macro(macro_kind));
+                self.tables.opt_def_kind.set(def_id.index, DefKind::Macro(macro_kind));
                 record!(self.tables.kind[def_id] <- EntryKind::ProcMacro(macro_kind));
                 record!(self.tables.attributes[def_id] <- attrs);
                 record!(self.tables.def_keys[def_id] <- def_key);
@@ -1886,14 +1884,14 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
 
         match nitem.kind {
             hir::ForeignItemKind::Fn(_, ref names, _) => {
-                record!(self.tables.asyncness[def_id] <- hir::IsAsync::NotAsync);
+                self.tables.asyncness.set(def_id.index, hir::IsAsync::NotAsync);
                 record!(self.tables.fn_arg_names[def_id] <- *names);
                 let constness = if self.tcx.is_const_fn_raw(def_id) {
                     hir::Constness::Const
                 } else {
                     hir::Constness::NotConst
                 };
-                record!(self.tables.impl_constness[def_id] <- constness);
+                self.tables.impl_constness.set(def_id.index, constness);
                 record!(self.tables.kind[def_id] <- EntryKind::ForeignFn);
             }
             hir::ForeignItemKind::Static(..) => {

compiler/rustc_metadata/src/rmeta/mod.rs

@@ -1,3 +1,4 @@
+use crate::creader::CrateMetadataRef;
 use decoder::Metadata;
 use def_path_hash_map::DefPathHashMapRef;
 use table::{Table, TableBuilder};
@@ -8,7 +9,7 @@ use rustc_data_structures::svh::Svh;
 use rustc_data_structures::sync::MetadataRef;
 use rustc_hir as hir;
 use rustc_hir::def::{CtorKind, DefKind};
-use rustc_hir::def_id::{DefId, DefIndex, DefPathHash, StableCrateId};
+use rustc_hir::def_id::{CrateNum, DefId, DefIndex, DefPathHash, StableCrateId};
 use rustc_hir::definitions::DefKey;
 use rustc_hir::lang_items;
 use rustc_index::{bit_set::FiniteBitSet, vec::IndexVec};
@@ -237,6 +238,29 @@ crate struct CrateRoot<'tcx> {
     symbol_mangling_version: SymbolManglingVersion,
 }
 
+/// On-disk representation of `DefId`.
+/// This creates a type-safe way to enforce that we remap the CrateNum between the on-disk
+/// representation and the compilation session.
+#[derive(Copy, Clone)]
+crate struct RawDefId {
+    krate: u32,
+    index: u32,
+}
+
+impl Into<RawDefId> for DefId {
+    fn into(self) -> RawDefId {
+        RawDefId { krate: self.krate.as_u32(), index: self.index.as_u32() }
+    }
+}
+
+impl RawDefId {
+    fn decode(self, cdata: CrateMetadataRef<'_>) -> DefId {
+        let krate = CrateNum::from_u32(self.krate);
+        let krate = cdata.map_encoded_cnum_to_current(krate);
+        DefId { krate, index: DefIndex::from_u32(self.index) }
+    }
+}
+
 #[derive(Encodable, Decodable)]
 crate struct CrateDep {
     pub name: Symbol,
@@ -286,7 +310,7 @@ define_tables! {
     attributes: Table<DefIndex, Lazy<[ast::Attribute]>>,
     children: Table<DefIndex, Lazy<[DefIndex]>>,
 
-    opt_def_kind: Table<DefIndex, Lazy<DefKind>>,
+    opt_def_kind: Table<DefIndex, DefKind>,
     visibility: Table<DefIndex, Lazy<ty::Visibility>>,
     def_span: Table<DefIndex, Lazy<Span>>,
     def_ident_span: Table<DefIndex, Lazy<Span>>,
@@ -309,20 +333,20 @@ define_tables! {
     mir_for_ctfe: Table<DefIndex, Lazy!(mir::Body<'tcx>)>,
     promoted_mir: Table<DefIndex, Lazy!(IndexVec<mir::Promoted, mir::Body<'tcx>>)>,
     thir_abstract_const: Table<DefIndex, Lazy!(&'tcx [thir::abstract_const::Node<'tcx>])>,
-    impl_parent: Table<DefIndex, Lazy!(DefId)>,
-    impl_polarity: Table<DefIndex, Lazy!(ty::ImplPolarity)>,
-    impl_constness: Table<DefIndex, Lazy!(hir::Constness)>,
-    impl_defaultness: Table<DefIndex, Lazy!(hir::Defaultness)>,
+    impl_parent: Table<DefIndex, RawDefId>,
+    impl_polarity: Table<DefIndex, ty::ImplPolarity>,
+    impl_constness: Table<DefIndex, hir::Constness>,
+    impl_defaultness: Table<DefIndex, hir::Defaultness>,
     // FIXME(eddyb) perhaps compute this on the fly if cheap enough?
     coerce_unsized_info: Table<DefIndex, Lazy!(ty::adjustment::CoerceUnsizedInfo)>,
     mir_const_qualif: Table<DefIndex, Lazy!(mir::ConstQualifs)>,
     rendered_const: Table<DefIndex, Lazy!(String)>,
-    asyncness: Table<DefIndex, Lazy!(hir::IsAsync)>,
+    asyncness: Table<DefIndex, hir::IsAsync>,
     fn_arg_names: Table<DefIndex, Lazy!([Ident])>,
     generator_kind: Table<DefIndex, Lazy!(hir::GeneratorKind)>,
     trait_def: Table<DefIndex, Lazy!(ty::TraitDef)>,
 
-    trait_item_def_id: Table<DefIndex, Lazy<DefId>>,
+    trait_item_def_id: Table<DefIndex, RawDefId>,
     inherent_impls: Table<DefIndex, Lazy<[DefIndex]>>,
     expn_that_defined: Table<DefIndex, Lazy<ExpnId>>,
     unused_generic_params: Table<DefIndex, Lazy<FiniteBitSet<u32>>>,
@@ -332,7 +356,7 @@ define_tables! {
     // `DefPathTable` up front, since we may only ever use a few
     // definitions from any given crate.
     def_keys: Table<DefIndex, Lazy<DefKey>>,
-    def_path_hashes: Table<DefIndex, Lazy<DefPathHash>>,
+    def_path_hashes: Table<DefIndex, DefPathHash>,
     proc_macro_quoted_spans: Table<usize, Lazy<Span>>,
 }