[perf] Decouple directly accessing a HIR owner from ast lowering

compiler/rustc_ast_lowering/src/lib.rs

@@ -54,8 +54,8 @@ use rustc_errors::{DiagArgFromDisplay, DiagCtxtHandle, StashKey};
 use rustc_hir::def::{DefKind, LifetimeRes, Namespace, PartialRes, PerNS, Res};
 use rustc_hir::def_id::{CRATE_DEF_ID, LOCAL_CRATE, LocalDefId};
 use rustc_hir::{
-    self as hir, AngleBrackets, ConstArg, GenericArg, HirId, ItemLocalMap, LangItem,
-    LifetimeSource, LifetimeSyntax, ParamName, TraitCandidate,
+    self as hir, AngleBrackets, AttributeMap, ConstArg, GenericArg, HirId, ItemLocalMap, LangItem,
+    LifetimeSource, LifetimeSyntax, MaybeOwner, OwnerId, ParamName, TraitCandidate,
 };
 use rustc_index::{Idx, IndexSlice, IndexVec};
 use rustc_macros::extension;
@@ -429,7 +429,7 @@ fn compute_hir_hash(
     })
 }
 
-pub fn lower_to_hir(tcx: TyCtxt<'_>, (): ()) -> hir::Crate<'_> {
+pub fn lower_to_hir(tcx: TyCtxt<'_>, (): ()) -> hir::Crate {
     let sess = tcx.sess;
     // Queries that borrow `resolver_for_lowering`.
     tcx.ensure_done().output_filenames(());
@@ -466,7 +466,27 @@ pub fn lower_to_hir(tcx: TyCtxt<'_>, (): ()) -> hir::Crate<'_> {
     // Don't hash unless necessary, because it's expensive.
     let opt_hir_hash =
         if tcx.needs_crate_hash() { Some(compute_hir_hash(tcx, &owners)) } else { None };
-    hir::Crate { owners, opt_hir_hash }
+
+    for (def_id, owner) in owners.drain_enumerated(..) {
+        let feed = tcx.super_duper_perf_hack_experiment(def_id);
+        feed.hir_owner(owner);
+        tcx.super_duper_perf_hack_experiment2(OwnerId { def_id }).hir_attr_map(
+            match owner.as_owner() {
+                Some(o) => &o.attrs,
+                None => AttributeMap::EMPTY,
+            },
+        );
+        feed.opt_hir_owner_nodes(owner.as_owner().map(|i| &i.nodes));
+        match owner {
+            MaybeOwner::Owner(_) => feed.local_def_id_to_hir_id(HirId::make_owner(def_id)),
+            MaybeOwner::NonOwner(hir_id) => feed.local_def_id_to_hir_id(hir_id),
+            MaybeOwner::Phantom => {}
+        }
+        tcx.super_duper_perf_hack_experiment2(OwnerId { def_id })
+            .in_scope_traits_map(owner.as_owner().map(|owner_info| &owner_info.trait_map));
+    }
+
+    hir::Crate { opt_hir_hash }
 }
 
 #[derive(Copy, Clone, PartialEq, Debug)]

compiler/rustc_driver_impl/src/pretty.rs

@@ -292,7 +292,11 @@ pub fn print<'tcx>(sess: &Session, ppm: PpMode, ex: PrintExtra<'tcx>) {
         }
         HirTree => {
             debug!("pretty printing HIR tree");
-            format!("{:#?}", ex.tcx().hir_crate(()))
+            let mut s = String::new();
+            for id in ex.tcx().hir_crate_items(()).owners() {
+                writeln!(&mut s, "{:#?}", ex.tcx().opt_hir_owner_nodes(id.def_id)).unwrap();
+            }
+            s
         }
         Mir => {
             let mut out = Vec::new();

compiler/rustc_hir/src/definitions.rs

@@ -88,6 +88,10 @@ impl DefPathTable {
         DefPathHash::new(self.stable_crate_id, hash)
     }
 
+    pub fn def_keys(&self) -> impl Iterator<Item = DefIndex> + ExactSizeIterator {
+        self.index_to_key.iter_enumerated().map(|(index, _)| index)
+    }
+
     pub fn enumerated_keys_and_path_hashes(
         &self,
     ) -> impl Iterator<Item = (DefIndex, &DefKey, DefPathHash)> + ExactSizeIterator {

compiler/rustc_hir/src/hir.rs

@@ -1462,11 +1462,15 @@ pub struct AttributeMap<'tcx> {
 }
 
 impl<'tcx> AttributeMap<'tcx> {
-    pub const EMPTY: &'static AttributeMap<'static> = &AttributeMap {
-        map: SortedMap::new(),
-        opt_hash: Some(Fingerprint::ZERO),
-        define_opaque: None,
-    };
+    pub const EMPTY: &'static AttributeMap<'static> = &Self::empty();
+
+    pub const fn empty() -> AttributeMap<'static> {
+        AttributeMap {
+            map: SortedMap::new(),
+            opt_hash: Some(Fingerprint::ZERO),
+            define_opaque: None,
+        }
+    }
 
     #[inline]
     pub fn get(&self, id: ItemLocalId) -> &'tcx [Attribute] {
@@ -1567,8 +1571,7 @@ impl<'tcx> MaybeOwner<'tcx> {
 ///
 /// [rustc dev guide]: https://rustc-dev-guide.rust-lang.org/hir.html
 #[derive(Debug)]
-pub struct Crate<'hir> {
-    pub owners: IndexVec<LocalDefId, MaybeOwner<'hir>>,
+pub struct Crate {
     // Only present when incr. comp. is enabled.
     pub opt_hir_hash: Option<Fingerprint>,
 }

compiler/rustc_hir/src/stable_hash_impls.rs

@@ -111,9 +111,9 @@ impl<'tcx, HirCtx: crate::HashStableContext> HashStable<HirCtx> for AttributeMap
     }
 }
 
-impl<HirCtx: crate::HashStableContext> HashStable<HirCtx> for Crate<'_> {
+impl<HirCtx: crate::HashStableContext> HashStable<HirCtx> for Crate {
     fn hash_stable(&self, hcx: &mut HirCtx, hasher: &mut StableHasher) {
-        let Crate { owners: _, opt_hir_hash } = self;
+        let Crate { opt_hir_hash } = self;
         opt_hir_hash.unwrap().hash_stable(hcx, hasher)
     }
 }

compiler/rustc_interface/src/passes.rs

@@ -928,16 +928,16 @@ fn run_required_analyses(tcx: TyCtxt<'_>) {
     if tcx.sess.opts.unstable_opts.input_stats {
         rustc_passes::input_stats::print_hir_stats(tcx);
     }
+
+    // Prefetch this to prevent multiple threads from blocking on it later.
+    // This also feeds all the queries that normally depend on hir_crate having been run.
+    tcx.ensure_done().hir_crate(());
+
     // When using rustdoc's "jump to def" feature, it enters this code and `check_crate`
     // is not defined. So we need to cfg it out.
     #[cfg(all(not(doc), debug_assertions))]
     rustc_passes::hir_id_validator::check_crate(tcx);
 
-    // Prefetch this to prevent multiple threads from blocking on it later.
-    // This is needed since the `hir_id_validator::check_crate` call above is not guaranteed
-    // to use `hir_crate`.
-    tcx.ensure_done().hir_crate(());
-
     let sess = tcx.sess;
     sess.time("misc_checking_1", || {
         parallel!(

compiler/rustc_middle/src/arena.rs

@@ -118,6 +118,7 @@ macro_rules! arena_types {
             [decode] specialization_graph: rustc_middle::traits::specialization_graph::Graph,
             [] crate_inherent_impls: rustc_middle::ty::CrateInherentImpls,
             [] hir_owner_nodes: rustc_hir::OwnerNodes<'tcx>,
+            [] owner_info: rustc_hir::OwnerInfo<'tcx>,
         ]);
     )
 }

compiler/rustc_middle/src/hir/map.rs

@@ -396,12 +396,9 @@ impl<'tcx> TyCtxt<'tcx> {
     where
         V: Visitor<'tcx>,
     {
-        let krate = self.hir_crate(());
-        for info in krate.owners.iter() {
-            if let MaybeOwner::Owner(info) = info {
-                for attrs in info.attrs.map.values() {
-                    walk_list!(visitor, visit_attribute, *attrs);
-                }
+        for owner in self.hir_crate_items(()).owners().chain(Some(CRATE_OWNER_ID)) {
+            for attrs in self.hir_attr_map(owner).map.values() {
+                walk_list!(visitor, visit_attribute, *attrs);
             }
         }
         V::Result::output()

compiler/rustc_middle/src/hir/mod.rs

@@ -181,19 +181,25 @@ pub fn provide(providers: &mut Providers) {
     providers.hir_crate_items = map::hir_crate_items;
     providers.crate_hash = map::crate_hash;
     providers.hir_module_items = map::hir_module_items;
-    providers.local_def_id_to_hir_id = |tcx, def_id| match tcx.hir_crate(()).owners[def_id] {
-        MaybeOwner::Owner(_) => HirId::make_owner(def_id),
-        MaybeOwner::NonOwner(hir_id) => hir_id,
-        MaybeOwner::Phantom => bug!("No HirId for {:?}", def_id),
+    providers.local_def_id_to_hir_id = |tcx, def_id| {
+        tcx.ensure_ok().hir_crate(());
+        match tcx.hir_owner(def_id) {
+            MaybeOwner::Owner(_) => HirId::make_owner(def_id),
+            MaybeOwner::NonOwner(hir_id) => hir_id,
+            MaybeOwner::Phantom => bug!("No HirId for {:?}", def_id),
+        }
+    };
+    providers.opt_hir_owner_nodes = |tcx, id| {
+        tcx.ensure_ok().hir_crate(());
+        tcx.hir_owner(id).as_owner().map(|i| &i.nodes)
     };
-    providers.opt_hir_owner_nodes =
-        |tcx, id| tcx.hir_crate(()).owners.get(id)?.as_owner().map(|i| &i.nodes);
     providers.hir_owner_parent = |tcx, owner_id| {
         tcx.opt_local_parent(owner_id.def_id).map_or(CRATE_HIR_ID, |parent_def_id| {
             let parent_owner_id = tcx.local_def_id_to_hir_id(parent_def_id).owner;
             HirId {
                 owner: parent_owner_id,
-                local_id: tcx.hir_crate(()).owners[parent_owner_id.def_id]
+                local_id: tcx
+                    .hir_owner(parent_owner_id.def_id)
                     .unwrap()
                     .parenting
                     .get(&owner_id.def_id)
@@ -203,7 +209,8 @@ pub fn provide(providers: &mut Providers) {
         })
     };
     providers.hir_attr_map = |tcx, id| {
-        tcx.hir_crate(()).owners[id.def_id].as_owner().map_or(AttributeMap::EMPTY, |o| &o.attrs)
+        tcx.ensure_ok().hir_crate(());
+        tcx.hir_owner(id.def_id).as_owner().map_or(AttributeMap::EMPTY, |o| &o.attrs)
     };
     providers.def_span = |tcx, def_id| tcx.hir_span(tcx.local_def_id_to_hir_id(def_id));
     providers.def_ident_span = |tcx, def_id| {
@@ -236,7 +243,6 @@ pub fn provide(providers: &mut Providers) {
         |tcx, trait_id| tcx.resolutions(()).trait_impls.get(&trait_id).map_or(&[], |xs| &xs[..]);
     providers.expn_that_defined =
         |tcx, id| tcx.resolutions(()).expn_that_defined.get(&id).copied().unwrap_or(ExpnId::root());
-    providers.in_scope_traits_map = |tcx, id| {
-        tcx.hir_crate(()).owners[id.def_id].as_owner().map(|owner_info| &owner_info.trait_map)
-    };
+    providers.in_scope_traits_map =
+        |tcx, id| tcx.hir_owner(id.def_id).as_owner().map(|owner_info| &owner_info.trait_map);
 }

compiler/rustc_middle/src/query/mod.rs

@@ -166,12 +166,19 @@ rustc_queries! {
     /// query gives you access to all other items. To avoid this fate, do not
     /// call `tcx.hir_crate(())`; instead, prefer wrappers like
     /// [`TyCtxt::hir_visit_all_item_likes_in_crate`].
-    query hir_crate(key: ()) -> &'tcx Crate<'tcx> {
+    query hir_crate(key: ()) -> &'tcx Crate {
         arena_cache
         eval_always
         desc { "getting the crate HIR" }
     }
 
+    /// A query decoupling the `hir_crate` query from everything else
+    query hir_owner(key: LocalDefId) -> rustc_hir::MaybeOwner<'tcx> {
+        no_hash
+        desc { |tcx| "getting HIR of `{}`", tcx.def_path_str(key) }
+        feedable
+    }
+
     /// All items in the crate.
     query hir_crate_items(_: ()) -> &'tcx rustc_middle::hir::ModuleItems {
         arena_cache
@@ -201,6 +208,7 @@ rustc_queries! {
     /// Avoid calling this query directly.
     query hir_owner_parent(key: hir::OwnerId) -> hir::HirId {
         desc { |tcx| "getting HIR parent of `{}`", tcx.def_path_str(key) }
+        feedable
     }
 
     /// Gives access to the HIR nodes and bodies inside `key` if it's a HIR owner.
@@ -1790,6 +1798,7 @@ rustc_queries! {
     query in_scope_traits_map(_: hir::OwnerId)
         -> Option<&'tcx ItemLocalMap<Box<[TraitCandidate]>>> {
         desc { "getting traits in scope at a block" }
+        feedable
     }
 
     /// Returns whether the impl or associated function has the `default` keyword.

compiler/rustc_middle/src/ty/context.rs

@@ -37,7 +37,9 @@ use rustc_hir::def_id::{CrateNum, DefId, LOCAL_CRATE, LocalDefId};
 use rustc_hir::definitions::{DefPathData, Definitions, DisambiguatorState};
 use rustc_hir::intravisit::VisitorExt;
 use rustc_hir::lang_items::LangItem;
-use rustc_hir::{self as hir, Attribute, HirId, Node, TraitCandidate};
+use rustc_hir::{
+    self as hir, Attribute, HirId, MaybeOwner, Node, OwnerId, OwnerInfo, TraitCandidate,
+};
 use rustc_index::IndexVec;
 use rustc_macros::{HashStable, TyDecodable, TyEncodable};
 use rustc_query_system::cache::WithDepNode;
@@ -1275,6 +1277,16 @@ impl<'tcx> TyCtxt<'tcx> {
         TyCtxtFeed { tcx: self, key: () }
     }
 
+    pub fn super_duper_perf_hack_experiment(self, key: LocalDefId) -> TyCtxtFeed<'tcx, LocalDefId> {
+        self.dep_graph.assert_eval_always();
+        TyCtxtFeed { tcx: self, key }
+    }
+
+    pub fn super_duper_perf_hack_experiment2(self, key: OwnerId) -> TyCtxtFeed<'tcx, OwnerId> {
+        self.dep_graph.assert_eval_always();
+        TyCtxtFeed { tcx: self, key }
+    }
+
     /// Only used in the resolver to register the `CRATE_DEF_ID` `DefId` and feed
     /// some queries for it. It will panic if used twice.
     pub fn create_local_crate_def_id(self, span: Span) -> TyCtxtFeed<'tcx, LocalDefId> {
@@ -1329,24 +1341,25 @@ impl<'tcx> TyCtxtFeed<'tcx, LocalDefId> {
 
     // Fills in all the important parts needed by HIR queries
     pub fn feed_hir(&self) {
-        self.local_def_id_to_hir_id(HirId::make_owner(self.def_id()));
-
         let node = hir::OwnerNode::Synthetic;
         let bodies = Default::default();
-        let attrs = hir::AttributeMap::EMPTY;
-
+        let attrs = hir::AttributeMap::empty();
         let (opt_hash_including_bodies, _) =
             self.tcx.hash_owner_nodes(node, &bodies, &attrs.map, attrs.define_opaque);
         let node = node.into();
-        self.opt_hir_owner_nodes(Some(self.tcx.arena.alloc(hir::OwnerNodes {
-            opt_hash_including_bodies,
-            nodes: IndexVec::from_elem_n(
-                hir::ParentedNode { parent: hir::ItemLocalId::INVALID, node },
-                1,
-            ),
-            bodies,
+        self.hir_owner(MaybeOwner::Owner(self.tcx.arena.alloc(OwnerInfo {
+            nodes: hir::OwnerNodes {
+                opt_hash_including_bodies,
+                nodes: IndexVec::from_elem_n(
+                    hir::ParentedNode { parent: hir::ItemLocalId::INVALID, node },
+                    1,
+                ),
+                bodies,
+            },
+            parenting: Default::default(),
+            attrs,
+            trait_map: Default::default(),
         })));
-        self.feed_owner_id().hir_attr_map(attrs);
     }
 }
 

compiler/rustc_query_system/src/dep_graph/graph.rs

@@ -200,6 +200,18 @@ impl<D: Deps> DepGraph<D> {
         }
     }
 
+    pub fn assert_eval_always(&self) {
+        if let Some(..) = self.data {
+            D::read_deps(|task_deps| {
+                assert_matches!(
+                    task_deps,
+                    TaskDepsRef::EvalAlways,
+                    "expected no task dependency tracking"
+                );
+            })
+        }
+    }
+
     pub fn with_ignore<OP, R>(&self, op: OP) -> R
     where
         OP: FnOnce() -> R,