diff --git a/crates/rust-analyzer/src/global_state.rs b/crates/rust-analyzer/src/global_state.rs
index 1b4c33d85868..8516ffa0dfae 100644
--- a/crates/rust-analyzer/src/global_state.rs
+++ b/crates/rust-analyzer/src/global_state.rs
@@ -307,16 +307,18 @@ impl GlobalState {
             for file in changed_files {
                 let vfs_path = vfs.file_path(file.file_id);
                 if let Some(path) = vfs_path.as_path() {
-                    let path = path.to_path_buf();
-                    if reload::should_refresh_for_change(&path, file.kind()) {
-                        workspace_structure_change = Some((path.clone(), false));
+                    has_structure_changes = file.is_created_or_deleted();
+
+                    if file.is_modified() && path.extension() == Some("rs") {
+                        modified_rust_files.push(file.file_id);
                     }
+
+                    let path = path.to_path_buf();
                     if file.is_created_or_deleted() {
-                        has_structure_changes = true;
-                        workspace_structure_change =
-                            Some((path, self.crate_graph_file_dependencies.contains(vfs_path)));
-                    } else if path.extension() == Some("rs".as_ref()) {
-                        modified_rust_files.push(file.file_id);
+                        workspace_structure_change.get_or_insert((path, false)).1 |=
+                            self.crate_graph_file_dependencies.contains(vfs_path);
+                    } else if reload::should_refresh_for_change(&path, file.kind()) {
+                        workspace_structure_change.get_or_insert((path.clone(), false));
                     }
                 }
 
diff --git a/crates/rust-analyzer/src/reload.rs b/crates/rust-analyzer/src/reload.rs
index 967904a01aea..507f584ff77d 100644
--- a/crates/rust-analyzer/src/reload.rs
+++ b/crates/rust-analyzer/src/reload.rs
@@ -528,21 +528,11 @@ impl GlobalState {
         let (crate_graph, proc_macro_paths, layouts, toolchains) = {
             // Create crate graph from all the workspaces
             let vfs = &mut self.vfs.write().0;
-            let loader = &mut self.loader;
 
             let load = |path: &AbsPath| {
-                let _p = tracing::span!(tracing::Level::DEBUG, "switch_workspaces::load").entered();
                 let vfs_path = vfs::VfsPath::from(path.to_path_buf());
                 crate_graph_file_dependencies.insert(vfs_path.clone());
-                match vfs.file_id(&vfs_path) {
-                    Some(file_id) => Some(file_id),
-                    None => {
-                        // FIXME: Consider not loading this here?
-                        let contents = loader.handle.load_sync(path);
-                        vfs.set_file_contents(vfs_path.clone(), contents);
-                        vfs.file_id(&vfs_path)
-                    }
-                }
+                vfs.file_id(&vfs_path)
             };
 
             ws_to_crate_graph(&self.workspaces, self.config.extra_env(), load)
diff --git a/crates/vfs/src/lib.rs b/crates/vfs/src/lib.rs
index 824ce3987033..9d723f8143b6 100644
--- a/crates/vfs/src/lib.rs
+++ b/crates/vfs/src/lib.rs
@@ -121,6 +121,11 @@ impl ChangedFile {
         matches!(self.change, Change::Create(_) | Change::Delete)
     }
 
+    /// Returns `true` if the change is [`Modify`](ChangeKind::Modify).
+    pub fn is_modified(&self) -> bool {
+        matches!(self.change, Change::Modify(_))
+    }
+
     pub fn kind(&self) -> ChangeKind {
         match self.change {
             Change::Create(_) => ChangeKind::Create,