From 315b0a7254974f1a2ef9777944531cdbca15be7b Mon Sep 17 00:00:00 2001 From: Amos Wenger Date: Wed, 20 Jul 2022 18:27:58 +0200 Subject: [PATCH 01/18] Add sysroot-abi feature, copy 1.64 ABI fo rnow --- crates/proc-macro-srv/Cargo.toml | 3 + .../src/abis/abi_sysroot/mod.rs | 102 +++ .../src/abis/abi_sysroot/ra_server.rs | 792 ++++++++++++++++++ crates/proc-macro-srv/src/abis/mod.rs | 2 + crates/proc-macro-srv/src/lib.rs | 4 + crates/rust-analyzer/Cargo.toml | 1 + 6 files changed, 904 insertions(+) create mode 100644 crates/proc-macro-srv/src/abis/abi_sysroot/mod.rs create mode 100644 crates/proc-macro-srv/src/abis/abi_sysroot/ra_server.rs diff --git a/crates/proc-macro-srv/Cargo.toml b/crates/proc-macro-srv/Cargo.toml index ce1fa0063f8e..e39026ac70bf 100644 --- a/crates/proc-macro-srv/Cargo.toml +++ b/crates/proc-macro-srv/Cargo.toml @@ -30,3 +30,6 @@ expect-test = "1.4.0" # used as proc macro test targets proc-macro-test = { path = "../proc-macro-test" } + +[features] +sysroot-abi = [] diff --git a/crates/proc-macro-srv/src/abis/abi_sysroot/mod.rs b/crates/proc-macro-srv/src/abis/abi_sysroot/mod.rs new file mode 100644 index 000000000000..44712f419191 --- /dev/null +++ b/crates/proc-macro-srv/src/abis/abi_sysroot/mod.rs @@ -0,0 +1,102 @@ +//! Proc macro ABI + +extern crate proc_macro; + +#[allow(dead_code)] +#[doc(hidden)] +mod ra_server; + +use libloading::Library; +use proc_macro_api::ProcMacroKind; + +use super::PanicMessage; + +pub(crate) struct Abi { + exported_macros: Vec, +} + +impl From for PanicMessage { + fn from(p: proc_macro::bridge::PanicMessage) -> Self { + Self { message: p.as_str().map(|s| s.to_string()) } + } +} + +impl Abi { + pub unsafe fn from_lib(lib: &Library, symbol_name: String) -> Result { + let macros: libloading::Symbol<'_, &&[proc_macro::bridge::client::ProcMacro]> = + lib.get(symbol_name.as_bytes())?; + Ok(Self { exported_macros: macros.to_vec() }) + } + + pub fn expand( + &self, + macro_name: &str, + macro_body: &tt::Subtree, + attributes: Option<&tt::Subtree>, + ) -> Result { + let parsed_body = ra_server::TokenStream::with_subtree(macro_body.clone()); + + let parsed_attributes = attributes.map_or(ra_server::TokenStream::new(), |attr| { + ra_server::TokenStream::with_subtree(attr.clone()) + }); + + for proc_macro in &self.exported_macros { + match proc_macro { + proc_macro::bridge::client::ProcMacro::CustomDerive { + trait_name, client, .. + } if *trait_name == macro_name => { + let res = client.run( + &proc_macro::bridge::server::SameThread, + ra_server::RustAnalyzer::default(), + parsed_body, + true, + ); + return res.map(|it| it.into_subtree()).map_err(PanicMessage::from); + } + proc_macro::bridge::client::ProcMacro::Bang { name, client } + if *name == macro_name => + { + let res = client.run( + &proc_macro::bridge::server::SameThread, + ra_server::RustAnalyzer::default(), + parsed_body, + true, + ); + return res.map(|it| it.into_subtree()).map_err(PanicMessage::from); + } + proc_macro::bridge::client::ProcMacro::Attr { name, client } + if *name == macro_name => + { + let res = client.run( + &proc_macro::bridge::server::SameThread, + ra_server::RustAnalyzer::default(), + parsed_attributes, + parsed_body, + true, + ); + return res.map(|it| it.into_subtree()).map_err(PanicMessage::from); + } + _ => continue, + } + } + + Err(proc_macro::bridge::PanicMessage::String("Nothing to expand".to_string()).into()) + } + + pub fn list_macros(&self) -> Vec<(String, ProcMacroKind)> { + self.exported_macros + .iter() + .map(|proc_macro| match proc_macro { + proc_macro::bridge::client::ProcMacro::CustomDerive { trait_name, .. } => { + (trait_name.to_string(), ProcMacroKind::CustomDerive) + } + proc_macro::bridge::client::ProcMacro::Bang { name, .. } => { + (name.to_string(), ProcMacroKind::FuncLike) + } + proc_macro::bridge::client::ProcMacro::Attr { name, .. } => { + (name.to_string(), ProcMacroKind::Attr) + } + }) + .collect() + } +} diff --git a/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server.rs b/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server.rs new file mode 100644 index 000000000000..00b9d9bd3d3e --- /dev/null +++ b/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server.rs @@ -0,0 +1,792 @@ +//! proc-macro server implementation +//! +//! Based on idea from +//! The lib-proc-macro server backend is `TokenStream`-agnostic, such that +//! we could provide any TokenStream implementation. +//! The original idea from fedochet is using proc-macro2 as backend, +//! we use tt instead for better integration with RA. +//! +//! FIXME: No span and source file information is implemented yet + +use super::proc_macro::bridge::{self, server}; + +use std::collections::HashMap; +use std::hash::Hash; +use std::iter::FromIterator; +use std::ops::Bound; +use std::{ascii, vec::IntoIter}; + +type Group = tt::Subtree; +type TokenTree = tt::TokenTree; +type Punct = tt::Punct; +type Spacing = tt::Spacing; +type Literal = tt::Literal; +type Span = tt::TokenId; + +#[derive(Debug, Default, Clone)] +pub struct TokenStream { + pub token_trees: Vec, +} + +impl TokenStream { + pub fn new() -> Self { + TokenStream::default() + } + + pub fn with_subtree(subtree: tt::Subtree) -> Self { + if subtree.delimiter.is_some() { + TokenStream { token_trees: vec![TokenTree::Subtree(subtree)] } + } else { + TokenStream { token_trees: subtree.token_trees } + } + } + + pub fn into_subtree(self) -> tt::Subtree { + tt::Subtree { delimiter: None, token_trees: self.token_trees } + } + + pub fn is_empty(&self) -> bool { + self.token_trees.is_empty() + } +} + +/// Creates a token stream containing a single token tree. +impl From for TokenStream { + fn from(tree: TokenTree) -> TokenStream { + TokenStream { token_trees: vec![tree] } + } +} + +/// Collects a number of token trees into a single stream. +impl FromIterator for TokenStream { + fn from_iter>(trees: I) -> Self { + trees.into_iter().map(TokenStream::from).collect() + } +} + +/// A "flattening" operation on token streams, collects token trees +/// from multiple token streams into a single stream. +impl FromIterator for TokenStream { + fn from_iter>(streams: I) -> Self { + let mut builder = TokenStreamBuilder::new(); + streams.into_iter().for_each(|stream| builder.push(stream)); + builder.build() + } +} + +impl Extend for TokenStream { + fn extend>(&mut self, trees: I) { + self.extend(trees.into_iter().map(TokenStream::from)); + } +} + +impl Extend for TokenStream { + fn extend>(&mut self, streams: I) { + for item in streams { + for tkn in item { + match tkn { + tt::TokenTree::Subtree(subtree) if subtree.delimiter.is_none() => { + self.token_trees.extend(subtree.token_trees); + } + _ => { + self.token_trees.push(tkn); + } + } + } + } + } +} + +#[derive(Clone)] +pub struct SourceFile { + // FIXME stub +} + +type Level = super::proc_macro::Level; +type LineColumn = super::proc_macro::LineColumn; + +/// A structure representing a diagnostic message and associated children +/// messages. +#[derive(Clone, Debug)] +pub struct Diagnostic { + level: Level, + message: String, + spans: Vec, + children: Vec, +} + +impl Diagnostic { + /// Creates a new diagnostic with the given `level` and `message`. + pub fn new>(level: Level, message: T) -> Diagnostic { + Diagnostic { level, message: message.into(), spans: vec![], children: vec![] } + } +} + +// Rustc Server Ident has to be `Copyable` +// We use a stub here for bypassing +#[derive(Hash, Eq, PartialEq, Copy, Clone)] +pub struct IdentId(u32); + +#[derive(Clone, Hash, Eq, PartialEq)] +struct IdentData(tt::Ident); + +#[derive(Default)] +struct IdentInterner { + idents: HashMap, + ident_data: Vec, +} + +impl IdentInterner { + fn intern(&mut self, data: &IdentData) -> u32 { + if let Some(index) = self.idents.get(data) { + return *index; + } + + let index = self.idents.len() as u32; + self.ident_data.push(data.clone()); + self.idents.insert(data.clone(), index); + index + } + + fn get(&self, index: u32) -> &IdentData { + &self.ident_data[index as usize] + } + + #[allow(unused)] + fn get_mut(&mut self, index: u32) -> &mut IdentData { + self.ident_data.get_mut(index as usize).expect("Should be consistent") + } +} + +pub struct TokenStreamBuilder { + acc: TokenStream, +} + +/// Public implementation details for the `TokenStream` type, such as iterators. +pub mod token_stream { + use std::str::FromStr; + + use super::{TokenStream, TokenTree}; + + /// An iterator over `TokenStream`'s `TokenTree`s. + /// The iteration is "shallow", e.g., the iterator doesn't recurse into delimited groups, + /// and returns whole groups as token trees. + impl IntoIterator for TokenStream { + type Item = TokenTree; + type IntoIter = super::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.token_trees.into_iter() + } + } + + type LexError = String; + + /// Attempts to break the string into tokens and parse those tokens into a token stream. + /// May fail for a number of reasons, for example, if the string contains unbalanced delimiters + /// or characters not existing in the language. + /// All tokens in the parsed stream get `Span::call_site()` spans. + /// + /// NOTE: some errors may cause panics instead of returning `LexError`. We reserve the right to + /// change these errors into `LexError`s later. + impl FromStr for TokenStream { + type Err = LexError; + + fn from_str(src: &str) -> Result { + let (subtree, _token_map) = + mbe::parse_to_token_tree(src).ok_or("Failed to parse from mbe")?; + + let subtree = subtree_replace_token_ids_with_unspecified(subtree); + Ok(TokenStream::with_subtree(subtree)) + } + } + + impl ToString for TokenStream { + fn to_string(&self) -> String { + tt::pretty(&self.token_trees) + } + } + + fn subtree_replace_token_ids_with_unspecified(subtree: tt::Subtree) -> tt::Subtree { + tt::Subtree { + delimiter: subtree + .delimiter + .map(|d| tt::Delimiter { id: tt::TokenId::unspecified(), ..d }), + token_trees: subtree + .token_trees + .into_iter() + .map(token_tree_replace_token_ids_with_unspecified) + .collect(), + } + } + + fn token_tree_replace_token_ids_with_unspecified(tt: tt::TokenTree) -> tt::TokenTree { + match tt { + tt::TokenTree::Leaf(leaf) => { + tt::TokenTree::Leaf(leaf_replace_token_ids_with_unspecified(leaf)) + } + tt::TokenTree::Subtree(subtree) => { + tt::TokenTree::Subtree(subtree_replace_token_ids_with_unspecified(subtree)) + } + } + } + + fn leaf_replace_token_ids_with_unspecified(leaf: tt::Leaf) -> tt::Leaf { + match leaf { + tt::Leaf::Literal(lit) => { + tt::Leaf::Literal(tt::Literal { id: tt::TokenId::unspecified(), ..lit }) + } + tt::Leaf::Punct(punct) => { + tt::Leaf::Punct(tt::Punct { id: tt::TokenId::unspecified(), ..punct }) + } + tt::Leaf::Ident(ident) => { + tt::Leaf::Ident(tt::Ident { id: tt::TokenId::unspecified(), ..ident }) + } + } + } +} + +impl TokenStreamBuilder { + fn new() -> TokenStreamBuilder { + TokenStreamBuilder { acc: TokenStream::new() } + } + + fn push(&mut self, stream: TokenStream) { + self.acc.extend(stream.into_iter()) + } + + fn build(self) -> TokenStream { + self.acc + } +} + +pub struct FreeFunctions; + +#[derive(Clone)] +pub struct TokenStreamIter { + trees: IntoIter, +} + +#[derive(Default)] +pub struct RustAnalyzer { + ident_interner: IdentInterner, + // FIXME: store span information here. +} + +impl server::Types for RustAnalyzer { + type FreeFunctions = FreeFunctions; + type TokenStream = TokenStream; + type Ident = IdentId; + type Literal = Literal; + type SourceFile = SourceFile; + type Diagnostic = Diagnostic; + type Span = Span; + type MultiSpan = Vec; +} + +impl server::FreeFunctions for RustAnalyzer { + fn track_env_var(&mut self, _var: &str, _value: Option<&str>) { + // FIXME: track env var accesses + // https://github.com/rust-lang/rust/pull/71858 + } + fn track_path(&mut self, _path: &str) {} +} + +impl server::TokenStream for RustAnalyzer { + fn is_empty(&mut self, stream: &Self::TokenStream) -> bool { + stream.is_empty() + } + fn from_str(&mut self, src: &str) -> Self::TokenStream { + use std::str::FromStr; + + Self::TokenStream::from_str(src).expect("cannot parse string") + } + fn to_string(&mut self, stream: &Self::TokenStream) -> String { + stream.to_string() + } + fn from_token_tree( + &mut self, + tree: bridge::TokenTree, + ) -> Self::TokenStream { + match tree { + bridge::TokenTree::Group(group) => { + let group = Group { + delimiter: delim_to_internal(group.delimiter), + token_trees: match group.stream { + Some(stream) => stream.into_iter().collect(), + None => Vec::new(), + }, + }; + let tree = TokenTree::from(group); + Self::TokenStream::from_iter(vec![tree]) + } + + bridge::TokenTree::Ident(IdentId(index)) => { + let IdentData(ident) = self.ident_interner.get(index).clone(); + let ident: tt::Ident = ident; + let leaf = tt::Leaf::from(ident); + let tree = TokenTree::from(leaf); + Self::TokenStream::from_iter(vec![tree]) + } + + bridge::TokenTree::Literal(literal) => { + let leaf = tt::Leaf::from(literal); + let tree = TokenTree::from(leaf); + Self::TokenStream::from_iter(vec![tree]) + } + + bridge::TokenTree::Punct(p) => { + let punct = tt::Punct { + char: p.ch as char, + spacing: if p.joint { Spacing::Joint } else { Spacing::Alone }, + id: p.span, + }; + let leaf = tt::Leaf::from(punct); + let tree = TokenTree::from(leaf); + Self::TokenStream::from_iter(vec![tree]) + } + } + } + + fn expand_expr(&mut self, self_: &Self::TokenStream) -> Result { + Ok(self_.clone()) + } + + fn concat_trees( + &mut self, + base: Option, + trees: Vec>, + ) -> Self::TokenStream { + let mut builder = TokenStreamBuilder::new(); + if let Some(base) = base { + builder.push(base); + } + for tree in trees { + builder.push(self.from_token_tree(tree)); + } + builder.build() + } + + fn concat_streams( + &mut self, + base: Option, + streams: Vec, + ) -> Self::TokenStream { + let mut builder = TokenStreamBuilder::new(); + if let Some(base) = base { + builder.push(base); + } + for stream in streams { + builder.push(stream); + } + builder.build() + } + + fn into_trees( + &mut self, + stream: Self::TokenStream, + ) -> Vec> { + stream + .into_iter() + .map(|tree| match tree { + tt::TokenTree::Leaf(tt::Leaf::Ident(ident)) => { + bridge::TokenTree::Ident(IdentId(self.ident_interner.intern(&IdentData(ident)))) + } + tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) => bridge::TokenTree::Literal(lit), + tt::TokenTree::Leaf(tt::Leaf::Punct(punct)) => { + bridge::TokenTree::Punct(bridge::Punct { + ch: punct.char as u8, + joint: punct.spacing == Spacing::Joint, + span: punct.id, + }) + } + tt::TokenTree::Subtree(subtree) => bridge::TokenTree::Group(bridge::Group { + delimiter: delim_to_external(subtree.delimiter), + stream: if subtree.token_trees.is_empty() { + None + } else { + Some(subtree.token_trees.into_iter().collect()) + }, + span: bridge::DelimSpan::from_single( + subtree.delimiter.map_or(Span::unspecified(), |del| del.id), + ), + }), + }) + .collect() + } +} + +fn delim_to_internal(d: bridge::Delimiter) -> Option { + let kind = match d { + bridge::Delimiter::Parenthesis => tt::DelimiterKind::Parenthesis, + bridge::Delimiter::Brace => tt::DelimiterKind::Brace, + bridge::Delimiter::Bracket => tt::DelimiterKind::Bracket, + bridge::Delimiter::None => return None, + }; + Some(tt::Delimiter { id: tt::TokenId::unspecified(), kind }) +} + +fn delim_to_external(d: Option) -> bridge::Delimiter { + match d.map(|it| it.kind) { + Some(tt::DelimiterKind::Parenthesis) => bridge::Delimiter::Parenthesis, + Some(tt::DelimiterKind::Brace) => bridge::Delimiter::Brace, + Some(tt::DelimiterKind::Bracket) => bridge::Delimiter::Bracket, + None => bridge::Delimiter::None, + } +} + +fn spacing_to_internal(spacing: bridge::Spacing) -> Spacing { + match spacing { + bridge::Spacing::Alone => Spacing::Alone, + bridge::Spacing::Joint => Spacing::Joint, + } +} + +fn spacing_to_external(spacing: Spacing) -> bridge::Spacing { + match spacing { + Spacing::Alone => bridge::Spacing::Alone, + Spacing::Joint => bridge::Spacing::Joint, + } +} + +impl server::Ident for RustAnalyzer { + fn new(&mut self, string: &str, span: Self::Span, _is_raw: bool) -> Self::Ident { + IdentId(self.ident_interner.intern(&IdentData(tt::Ident { text: string.into(), id: span }))) + } + + fn span(&mut self, ident: Self::Ident) -> Self::Span { + self.ident_interner.get(ident.0).0.id + } + fn with_span(&mut self, ident: Self::Ident, span: Self::Span) -> Self::Ident { + let data = self.ident_interner.get(ident.0); + let new = IdentData(tt::Ident { id: span, ..data.0.clone() }); + IdentId(self.ident_interner.intern(&new)) + } +} + +impl server::Literal for RustAnalyzer { + fn debug_kind(&mut self, _literal: &Self::Literal) -> String { + // r-a: debug_kind and suffix are unsupported; corresponding client code has been changed to not call these. + // They must still be present to be ABI-compatible and work with upstream proc_macro. + "".to_owned() + } + fn from_str(&mut self, s: &str) -> Result { + Ok(Literal { text: s.into(), id: tt::TokenId::unspecified() }) + } + fn symbol(&mut self, literal: &Self::Literal) -> String { + literal.text.to_string() + } + fn suffix(&mut self, _literal: &Self::Literal) -> Option { + None + } + + fn to_string(&mut self, literal: &Self::Literal) -> String { + literal.to_string() + } + + fn integer(&mut self, n: &str) -> Self::Literal { + let n = match n.parse::() { + Ok(n) => n.to_string(), + Err(_) => n.parse::().unwrap().to_string(), + }; + Literal { text: n.into(), id: tt::TokenId::unspecified() } + } + + fn typed_integer(&mut self, n: &str, kind: &str) -> Self::Literal { + macro_rules! def_suffixed_integer { + ($kind:ident, $($ty:ty),*) => { + match $kind { + $( + stringify!($ty) => { + let n: $ty = n.parse().unwrap(); + format!(concat!("{}", stringify!($ty)), n) + } + )* + _ => unimplemented!("unknown args for typed_integer: n {}, kind {}", n, $kind), + } + } + } + + let text = def_suffixed_integer! {kind, u8, u16, u32, u64, u128, usize, i8, i16, i32, i64, i128, isize}; + + Literal { text: text.into(), id: tt::TokenId::unspecified() } + } + + fn float(&mut self, n: &str) -> Self::Literal { + let n: f64 = n.parse().unwrap(); + let mut text = f64::to_string(&n); + if !text.contains('.') { + text += ".0" + } + Literal { text: text.into(), id: tt::TokenId::unspecified() } + } + + fn f32(&mut self, n: &str) -> Self::Literal { + let n: f32 = n.parse().unwrap(); + let text = format!("{}f32", n); + Literal { text: text.into(), id: tt::TokenId::unspecified() } + } + + fn f64(&mut self, n: &str) -> Self::Literal { + let n: f64 = n.parse().unwrap(); + let text = format!("{}f64", n); + Literal { text: text.into(), id: tt::TokenId::unspecified() } + } + + fn string(&mut self, string: &str) -> Self::Literal { + let mut escaped = String::new(); + for ch in string.chars() { + escaped.extend(ch.escape_debug()); + } + Literal { text: format!("\"{}\"", escaped).into(), id: tt::TokenId::unspecified() } + } + + fn character(&mut self, ch: char) -> Self::Literal { + Literal { text: format!("'{}'", ch).into(), id: tt::TokenId::unspecified() } + } + + fn byte_string(&mut self, bytes: &[u8]) -> Self::Literal { + let string = bytes + .iter() + .cloned() + .flat_map(ascii::escape_default) + .map(Into::::into) + .collect::(); + + Literal { text: format!("b\"{}\"", string).into(), id: tt::TokenId::unspecified() } + } + + fn span(&mut self, literal: &Self::Literal) -> Self::Span { + literal.id + } + + fn set_span(&mut self, literal: &mut Self::Literal, span: Self::Span) { + literal.id = span; + } + + fn subspan( + &mut self, + _literal: &Self::Literal, + _start: Bound, + _end: Bound, + ) -> Option { + // FIXME handle span + None + } +} + +impl server::SourceFile for RustAnalyzer { + // FIXME these are all stubs + fn eq(&mut self, _file1: &Self::SourceFile, _file2: &Self::SourceFile) -> bool { + true + } + fn path(&mut self, _file: &Self::SourceFile) -> String { + String::new() + } + fn is_real(&mut self, _file: &Self::SourceFile) -> bool { + true + } +} + +impl server::Diagnostic for RustAnalyzer { + fn new(&mut self, level: Level, msg: &str, spans: Self::MultiSpan) -> Self::Diagnostic { + let mut diag = Diagnostic::new(level, msg); + diag.spans = spans; + diag + } + + fn sub( + &mut self, + _diag: &mut Self::Diagnostic, + _level: Level, + _msg: &str, + _spans: Self::MultiSpan, + ) { + // FIXME handle diagnostic + // + } + + fn emit(&mut self, _diag: Self::Diagnostic) { + // FIXME handle diagnostic + // diag.emit() + } +} + +impl server::Span for RustAnalyzer { + fn debug(&mut self, span: Self::Span) -> String { + format!("{:?}", span.0) + } + fn source_file(&mut self, _span: Self::Span) -> Self::SourceFile { + SourceFile {} + } + fn save_span(&mut self, _span: Self::Span) -> usize { + // FIXME stub + 0 + } + fn recover_proc_macro_span(&mut self, _id: usize) -> Self::Span { + // FIXME stub + tt::TokenId::unspecified() + } + /// Recent feature, not yet in the proc_macro + /// + /// See PR: + /// https://github.com/rust-lang/rust/pull/55780 + fn source_text(&mut self, _span: Self::Span) -> Option { + None + } + + fn parent(&mut self, _span: Self::Span) -> Option { + // FIXME handle span + None + } + fn source(&mut self, span: Self::Span) -> Self::Span { + // FIXME handle span + span + } + fn start(&mut self, _span: Self::Span) -> LineColumn { + // FIXME handle span + LineColumn { line: 0, column: 0 } + } + fn end(&mut self, _span: Self::Span) -> LineColumn { + // FIXME handle span + LineColumn { line: 0, column: 0 } + } + fn join(&mut self, first: Self::Span, _second: Self::Span) -> Option { + // Just return the first span again, because some macros will unwrap the result. + Some(first) + } + fn resolved_at(&mut self, _span: Self::Span, _at: Self::Span) -> Self::Span { + // FIXME handle span + tt::TokenId::unspecified() + } + + fn after(&mut self, _self_: Self::Span) -> Self::Span { + tt::TokenId::unspecified() + } + + fn before(&mut self, _self_: Self::Span) -> Self::Span { + tt::TokenId::unspecified() + } +} + +impl server::MultiSpan for RustAnalyzer { + fn new(&mut self) -> Self::MultiSpan { + // FIXME handle span + vec![] + } + + fn push(&mut self, other: &mut Self::MultiSpan, span: Self::Span) { + //TODP + other.push(span) + } +} + +impl server::Server for RustAnalyzer { + fn globals(&mut self) -> bridge::ExpnGlobals { + bridge::ExpnGlobals { + def_site: Span::unspecified(), + call_site: Span::unspecified(), + mixed_site: Span::unspecified(), + } + } +} + +#[cfg(test)] +mod tests { + use super::super::proc_macro::bridge::server::Literal; + use super::*; + + #[test] + fn test_ra_server_literals() { + let mut srv = RustAnalyzer { ident_interner: IdentInterner::default() }; + assert_eq!(srv.integer("1234").text, "1234"); + + assert_eq!(srv.typed_integer("12", "u8").text, "12u8"); + assert_eq!(srv.typed_integer("255", "u16").text, "255u16"); + assert_eq!(srv.typed_integer("1234", "u32").text, "1234u32"); + assert_eq!(srv.typed_integer("15846685", "u64").text, "15846685u64"); + assert_eq!(srv.typed_integer("15846685258", "u128").text, "15846685258u128"); + assert_eq!(srv.typed_integer("156788984", "usize").text, "156788984usize"); + assert_eq!(srv.typed_integer("127", "i8").text, "127i8"); + assert_eq!(srv.typed_integer("255", "i16").text, "255i16"); + assert_eq!(srv.typed_integer("1234", "i32").text, "1234i32"); + assert_eq!(srv.typed_integer("15846685", "i64").text, "15846685i64"); + assert_eq!(srv.typed_integer("15846685258", "i128").text, "15846685258i128"); + assert_eq!(srv.float("0").text, "0.0"); + assert_eq!(srv.float("15684.5867").text, "15684.5867"); + assert_eq!(srv.f32("15684.58").text, "15684.58f32"); + assert_eq!(srv.f64("15684.58").text, "15684.58f64"); + + assert_eq!(srv.string("hello_world").text, "\"hello_world\""); + assert_eq!(srv.character('c').text, "'c'"); + assert_eq!(srv.byte_string(b"1234586\x88").text, "b\"1234586\\x88\""); + + // u128::max + assert_eq!( + srv.integer("340282366920938463463374607431768211455").text, + "340282366920938463463374607431768211455" + ); + // i128::min + assert_eq!( + srv.integer("-170141183460469231731687303715884105728").text, + "-170141183460469231731687303715884105728" + ); + } + + #[test] + fn test_ra_server_to_string() { + let s = TokenStream { + token_trees: vec![ + tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { + text: "struct".into(), + id: tt::TokenId::unspecified(), + })), + tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { + text: "T".into(), + id: tt::TokenId::unspecified(), + })), + tt::TokenTree::Subtree(tt::Subtree { + delimiter: Some(tt::Delimiter { + id: tt::TokenId::unspecified(), + kind: tt::DelimiterKind::Brace, + }), + token_trees: vec![], + }), + ], + }; + + assert_eq!(s.to_string(), "struct T {}"); + } + + #[test] + fn test_ra_server_from_str() { + use std::str::FromStr; + let subtree_paren_a = tt::TokenTree::Subtree(tt::Subtree { + delimiter: Some(tt::Delimiter { + id: tt::TokenId::unspecified(), + kind: tt::DelimiterKind::Parenthesis, + }), + token_trees: vec![tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { + text: "a".into(), + id: tt::TokenId::unspecified(), + }))], + }); + + let t1 = TokenStream::from_str("(a)").unwrap(); + assert_eq!(t1.token_trees.len(), 1); + assert_eq!(t1.token_trees[0], subtree_paren_a); + + let t2 = TokenStream::from_str("(a);").unwrap(); + assert_eq!(t2.token_trees.len(), 2); + assert_eq!(t2.token_trees[0], subtree_paren_a); + + let underscore = TokenStream::from_str("_").unwrap(); + assert_eq!( + underscore.token_trees[0], + tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { + text: "_".into(), + id: tt::TokenId::unspecified(), + })) + ); + } +} diff --git a/crates/proc-macro-srv/src/abis/mod.rs b/crates/proc-macro-srv/src/abis/mod.rs index ae45b34b7258..62479b7ccd6b 100644 --- a/crates/proc-macro-srv/src/abis/mod.rs +++ b/crates/proc-macro-srv/src/abis/mod.rs @@ -26,6 +26,8 @@ mod abi_1_58; mod abi_1_63; mod abi_1_64; +#[cfg(feature = "sysroot-abi")] +mod abi_sysroot; // Used by `test/utils.rs` #[cfg(test)] diff --git a/crates/proc-macro-srv/src/lib.rs b/crates/proc-macro-srv/src/lib.rs index ca7765082f7c..2ab6eba6f424 100644 --- a/crates/proc-macro-srv/src/lib.rs +++ b/crates/proc-macro-srv/src/lib.rs @@ -11,6 +11,10 @@ //! rustc rather than `unstable`. (Although in general ABI compatibility is still an issue)… #![warn(rust_2018_idioms, unused_lifetimes, semicolon_in_expressions_from_macros)] +#![cfg_attr( + feature = "sysroot-abi", + feature(proc_macro_internals, proc_macro_diagnostic, proc_macro_span) +)] #![allow(unreachable_pub)] mod dylib; diff --git a/crates/rust-analyzer/Cargo.toml b/crates/rust-analyzer/Cargo.toml index c37a0d17491d..41205f2584a9 100644 --- a/crates/rust-analyzer/Cargo.toml +++ b/crates/rust-analyzer/Cargo.toml @@ -84,3 +84,4 @@ mbe = { path = "../mbe" } [features] jemalloc = ["jemallocator", "profile/jemalloc"] force-always-assert = ["always-assert/force"] +in-rust-tree = ["proc-macro-srv/sysroot-abi"] From e009cdc8d9f155e2a5e99202a433b426f0ac2ee4 Mon Sep 17 00:00:00 2001 From: Amos Wenger Date: Wed, 20 Jul 2022 18:32:27 +0200 Subject: [PATCH 02/18] Move token_stream to separate module --- .../src/abis/abi_sysroot/ra_server.rs | 230 +----------------- .../abi_sysroot/ra_server/token_stream.rs | 182 ++++++++++++++ 2 files changed, 188 insertions(+), 224 deletions(-) create mode 100644 crates/proc-macro-srv/src/abis/abi_sysroot/ra_server/token_stream.rs diff --git a/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server.rs b/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server.rs index 00b9d9bd3d3e..4c9d3364ba7f 100644 --- a/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server.rs +++ b/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server.rs @@ -10,11 +10,14 @@ use super::proc_macro::bridge::{self, server}; +mod token_stream; +pub use token_stream::*; + +use std::ascii; use std::collections::HashMap; use std::hash::Hash; use std::iter::FromIterator; use std::ops::Bound; -use std::{ascii, vec::IntoIter}; type Group = tt::Subtree; type TokenTree = tt::TokenTree; @@ -23,80 +26,6 @@ type Spacing = tt::Spacing; type Literal = tt::Literal; type Span = tt::TokenId; -#[derive(Debug, Default, Clone)] -pub struct TokenStream { - pub token_trees: Vec, -} - -impl TokenStream { - pub fn new() -> Self { - TokenStream::default() - } - - pub fn with_subtree(subtree: tt::Subtree) -> Self { - if subtree.delimiter.is_some() { - TokenStream { token_trees: vec![TokenTree::Subtree(subtree)] } - } else { - TokenStream { token_trees: subtree.token_trees } - } - } - - pub fn into_subtree(self) -> tt::Subtree { - tt::Subtree { delimiter: None, token_trees: self.token_trees } - } - - pub fn is_empty(&self) -> bool { - self.token_trees.is_empty() - } -} - -/// Creates a token stream containing a single token tree. -impl From for TokenStream { - fn from(tree: TokenTree) -> TokenStream { - TokenStream { token_trees: vec![tree] } - } -} - -/// Collects a number of token trees into a single stream. -impl FromIterator for TokenStream { - fn from_iter>(trees: I) -> Self { - trees.into_iter().map(TokenStream::from).collect() - } -} - -/// A "flattening" operation on token streams, collects token trees -/// from multiple token streams into a single stream. -impl FromIterator for TokenStream { - fn from_iter>(streams: I) -> Self { - let mut builder = TokenStreamBuilder::new(); - streams.into_iter().for_each(|stream| builder.push(stream)); - builder.build() - } -} - -impl Extend for TokenStream { - fn extend>(&mut self, trees: I) { - self.extend(trees.into_iter().map(TokenStream::from)); - } -} - -impl Extend for TokenStream { - fn extend>(&mut self, streams: I) { - for item in streams { - for tkn in item { - match tkn { - tt::TokenTree::Subtree(subtree) if subtree.delimiter.is_none() => { - self.token_trees.extend(subtree.token_trees); - } - _ => { - self.token_trees.push(tkn); - } - } - } - } - } -} - #[derive(Clone)] pub struct SourceFile { // FIXME stub @@ -158,130 +87,21 @@ impl IdentInterner { } } -pub struct TokenStreamBuilder { - acc: TokenStream, -} - -/// Public implementation details for the `TokenStream` type, such as iterators. -pub mod token_stream { - use std::str::FromStr; - - use super::{TokenStream, TokenTree}; - - /// An iterator over `TokenStream`'s `TokenTree`s. - /// The iteration is "shallow", e.g., the iterator doesn't recurse into delimited groups, - /// and returns whole groups as token trees. - impl IntoIterator for TokenStream { - type Item = TokenTree; - type IntoIter = super::IntoIter; - - fn into_iter(self) -> Self::IntoIter { - self.token_trees.into_iter() - } - } - - type LexError = String; - - /// Attempts to break the string into tokens and parse those tokens into a token stream. - /// May fail for a number of reasons, for example, if the string contains unbalanced delimiters - /// or characters not existing in the language. - /// All tokens in the parsed stream get `Span::call_site()` spans. - /// - /// NOTE: some errors may cause panics instead of returning `LexError`. We reserve the right to - /// change these errors into `LexError`s later. - impl FromStr for TokenStream { - type Err = LexError; - - fn from_str(src: &str) -> Result { - let (subtree, _token_map) = - mbe::parse_to_token_tree(src).ok_or("Failed to parse from mbe")?; - - let subtree = subtree_replace_token_ids_with_unspecified(subtree); - Ok(TokenStream::with_subtree(subtree)) - } - } - - impl ToString for TokenStream { - fn to_string(&self) -> String { - tt::pretty(&self.token_trees) - } - } - - fn subtree_replace_token_ids_with_unspecified(subtree: tt::Subtree) -> tt::Subtree { - tt::Subtree { - delimiter: subtree - .delimiter - .map(|d| tt::Delimiter { id: tt::TokenId::unspecified(), ..d }), - token_trees: subtree - .token_trees - .into_iter() - .map(token_tree_replace_token_ids_with_unspecified) - .collect(), - } - } - - fn token_tree_replace_token_ids_with_unspecified(tt: tt::TokenTree) -> tt::TokenTree { - match tt { - tt::TokenTree::Leaf(leaf) => { - tt::TokenTree::Leaf(leaf_replace_token_ids_with_unspecified(leaf)) - } - tt::TokenTree::Subtree(subtree) => { - tt::TokenTree::Subtree(subtree_replace_token_ids_with_unspecified(subtree)) - } - } - } - - fn leaf_replace_token_ids_with_unspecified(leaf: tt::Leaf) -> tt::Leaf { - match leaf { - tt::Leaf::Literal(lit) => { - tt::Leaf::Literal(tt::Literal { id: tt::TokenId::unspecified(), ..lit }) - } - tt::Leaf::Punct(punct) => { - tt::Leaf::Punct(tt::Punct { id: tt::TokenId::unspecified(), ..punct }) - } - tt::Leaf::Ident(ident) => { - tt::Leaf::Ident(tt::Ident { id: tt::TokenId::unspecified(), ..ident }) - } - } - } -} - -impl TokenStreamBuilder { - fn new() -> TokenStreamBuilder { - TokenStreamBuilder { acc: TokenStream::new() } - } - - fn push(&mut self, stream: TokenStream) { - self.acc.extend(stream.into_iter()) - } - - fn build(self) -> TokenStream { - self.acc - } -} - pub struct FreeFunctions; -#[derive(Clone)] -pub struct TokenStreamIter { - trees: IntoIter, -} - #[derive(Default)] pub struct RustAnalyzer { - ident_interner: IdentInterner, // FIXME: store span information here. } impl server::Types for RustAnalyzer { type FreeFunctions = FreeFunctions; type TokenStream = TokenStream; - type Ident = IdentId; - type Literal = Literal; type SourceFile = SourceFile; + type MultiSpan = Vec; type Diagnostic = Diagnostic; type Span = Span; - type MultiSpan = Vec; + type Symbol = Symbol; } impl server::FreeFunctions for RustAnalyzer { @@ -693,46 +513,8 @@ impl server::Server for RustAnalyzer { #[cfg(test)] mod tests { - use super::super::proc_macro::bridge::server::Literal; use super::*; - #[test] - fn test_ra_server_literals() { - let mut srv = RustAnalyzer { ident_interner: IdentInterner::default() }; - assert_eq!(srv.integer("1234").text, "1234"); - - assert_eq!(srv.typed_integer("12", "u8").text, "12u8"); - assert_eq!(srv.typed_integer("255", "u16").text, "255u16"); - assert_eq!(srv.typed_integer("1234", "u32").text, "1234u32"); - assert_eq!(srv.typed_integer("15846685", "u64").text, "15846685u64"); - assert_eq!(srv.typed_integer("15846685258", "u128").text, "15846685258u128"); - assert_eq!(srv.typed_integer("156788984", "usize").text, "156788984usize"); - assert_eq!(srv.typed_integer("127", "i8").text, "127i8"); - assert_eq!(srv.typed_integer("255", "i16").text, "255i16"); - assert_eq!(srv.typed_integer("1234", "i32").text, "1234i32"); - assert_eq!(srv.typed_integer("15846685", "i64").text, "15846685i64"); - assert_eq!(srv.typed_integer("15846685258", "i128").text, "15846685258i128"); - assert_eq!(srv.float("0").text, "0.0"); - assert_eq!(srv.float("15684.5867").text, "15684.5867"); - assert_eq!(srv.f32("15684.58").text, "15684.58f32"); - assert_eq!(srv.f64("15684.58").text, "15684.58f64"); - - assert_eq!(srv.string("hello_world").text, "\"hello_world\""); - assert_eq!(srv.character('c').text, "'c'"); - assert_eq!(srv.byte_string(b"1234586\x88").text, "b\"1234586\\x88\""); - - // u128::max - assert_eq!( - srv.integer("340282366920938463463374607431768211455").text, - "340282366920938463463374607431768211455" - ); - // i128::min - assert_eq!( - srv.integer("-170141183460469231731687303715884105728").text, - "-170141183460469231731687303715884105728" - ); - } - #[test] fn test_ra_server_to_string() { let s = TokenStream { diff --git a/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server/token_stream.rs b/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server/token_stream.rs new file mode 100644 index 000000000000..9a31f2ebf87c --- /dev/null +++ b/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server/token_stream.rs @@ -0,0 +1,182 @@ +use tt::TokenTree; + +#[derive(Debug, Default, Clone)] +pub struct TokenStream { + pub token_trees: Vec, +} + +impl TokenStream { + pub fn new() -> Self { + TokenStream::default() + } + + pub fn with_subtree(subtree: tt::Subtree) -> Self { + if subtree.delimiter.is_some() { + TokenStream { token_trees: vec![TokenTree::Subtree(subtree)] } + } else { + TokenStream { token_trees: subtree.token_trees } + } + } + + pub fn into_subtree(self) -> tt::Subtree { + tt::Subtree { delimiter: None, token_trees: self.token_trees } + } + + pub fn is_empty(&self) -> bool { + self.token_trees.is_empty() + } +} + +/// Creates a token stream containing a single token tree. +impl From for TokenStream { + fn from(tree: TokenTree) -> TokenStream { + TokenStream { token_trees: vec![tree] } + } +} + +/// Collects a number of token trees into a single stream. +impl FromIterator for TokenStream { + fn from_iter>(trees: I) -> Self { + trees.into_iter().map(TokenStream::from).collect() + } +} + +/// A "flattening" operation on token streams, collects token trees +/// from multiple token streams into a single stream. +impl FromIterator for TokenStream { + fn from_iter>(streams: I) -> Self { + let mut builder = TokenStreamBuilder::new(); + streams.into_iter().for_each(|stream| builder.push(stream)); + builder.build() + } +} + +impl Extend for TokenStream { + fn extend>(&mut self, trees: I) { + self.extend(trees.into_iter().map(TokenStream::from)); + } +} + +impl Extend for TokenStream { + fn extend>(&mut self, streams: I) { + for item in streams { + for tkn in item { + match tkn { + tt::TokenTree::Subtree(subtree) if subtree.delimiter.is_none() => { + self.token_trees.extend(subtree.token_trees); + } + _ => { + self.token_trees.push(tkn); + } + } + } + } + } +} + +pub struct TokenStreamBuilder { + acc: TokenStream, +} + +/// Public implementation details for the `TokenStream` type, such as iterators. +pub mod token_stream { + use std::str::FromStr; + + use super::{TokenStream, TokenTree}; + + /// An iterator over `TokenStream`'s `TokenTree`s. + /// The iteration is "shallow", e.g., the iterator doesn't recurse into delimited groups, + /// and returns whole groups as token trees. + impl IntoIterator for TokenStream { + type Item = TokenTree; + type IntoIter = std::vec::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.token_trees.into_iter() + } + } + + type LexError = String; + + /// Attempts to break the string into tokens and parse those tokens into a token stream. + /// May fail for a number of reasons, for example, if the string contains unbalanced delimiters + /// or characters not existing in the language. + /// All tokens in the parsed stream get `Span::call_site()` spans. + /// + /// NOTE: some errors may cause panics instead of returning `LexError`. We reserve the right to + /// change these errors into `LexError`s later. + impl FromStr for TokenStream { + type Err = LexError; + + fn from_str(src: &str) -> Result { + let (subtree, _token_map) = + mbe::parse_to_token_tree(src).ok_or("Failed to parse from mbe")?; + + let subtree = subtree_replace_token_ids_with_unspecified(subtree); + Ok(TokenStream::with_subtree(subtree)) + } + } + + impl ToString for TokenStream { + fn to_string(&self) -> String { + tt::pretty(&self.token_trees) + } + } + + fn subtree_replace_token_ids_with_unspecified(subtree: tt::Subtree) -> tt::Subtree { + tt::Subtree { + delimiter: subtree + .delimiter + .map(|d| tt::Delimiter { id: tt::TokenId::unspecified(), ..d }), + token_trees: subtree + .token_trees + .into_iter() + .map(token_tree_replace_token_ids_with_unspecified) + .collect(), + } + } + + fn token_tree_replace_token_ids_with_unspecified(tt: tt::TokenTree) -> tt::TokenTree { + match tt { + tt::TokenTree::Leaf(leaf) => { + tt::TokenTree::Leaf(leaf_replace_token_ids_with_unspecified(leaf)) + } + tt::TokenTree::Subtree(subtree) => { + tt::TokenTree::Subtree(subtree_replace_token_ids_with_unspecified(subtree)) + } + } + } + + fn leaf_replace_token_ids_with_unspecified(leaf: tt::Leaf) -> tt::Leaf { + match leaf { + tt::Leaf::Literal(lit) => { + tt::Leaf::Literal(tt::Literal { id: tt::TokenId::unspecified(), ..lit }) + } + tt::Leaf::Punct(punct) => { + tt::Leaf::Punct(tt::Punct { id: tt::TokenId::unspecified(), ..punct }) + } + tt::Leaf::Ident(ident) => { + tt::Leaf::Ident(tt::Ident { id: tt::TokenId::unspecified(), ..ident }) + } + } + } +} + +impl TokenStreamBuilder { + fn new() -> TokenStreamBuilder { + TokenStreamBuilder { acc: TokenStream::new() } + } + + fn push(&mut self, stream: TokenStream) { + self.acc.extend(stream.into_iter()) + } + + fn build(self) -> TokenStream { + self.acc + } +} + +#[derive(Clone)] +pub struct TokenStreamIter { + trees: std::vec::IntoIter, +} From 00bc060ba311f5177f5f34424279e7f3000bde54 Mon Sep 17 00:00:00 2001 From: Amos Wenger Date: Wed, 20 Jul 2022 18:36:10 +0200 Subject: [PATCH 03/18] Fix imports, delete removed interfaces --- .../src/abis/abi_sysroot/ra_server.rs | 222 +++--------------- .../src/abis/abi_sysroot/ra_server/symbol.rs | 34 +++ 2 files changed, 66 insertions(+), 190 deletions(-) create mode 100644 crates/proc-macro-srv/src/abis/abi_sysroot/ra_server/symbol.rs diff --git a/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server.rs b/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server.rs index 4c9d3364ba7f..9265cd6b6b79 100644 --- a/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server.rs +++ b/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server.rs @@ -8,16 +8,18 @@ //! //! FIXME: No span and source file information is implemented yet -use super::proc_macro::bridge::{self, server}; +use super::proc_macro::{ + self, + bridge::{self, server}, +}; mod token_stream; pub use token_stream::*; -use std::ascii; -use std::collections::HashMap; -use std::hash::Hash; +mod symbol; +pub use symbol::*; + use std::iter::FromIterator; -use std::ops::Bound; type Group = tt::Subtree; type TokenTree = tt::TokenTree; @@ -51,42 +53,6 @@ impl Diagnostic { } } -// Rustc Server Ident has to be `Copyable` -// We use a stub here for bypassing -#[derive(Hash, Eq, PartialEq, Copy, Clone)] -pub struct IdentId(u32); - -#[derive(Clone, Hash, Eq, PartialEq)] -struct IdentData(tt::Ident); - -#[derive(Default)] -struct IdentInterner { - idents: HashMap, - ident_data: Vec, -} - -impl IdentInterner { - fn intern(&mut self, data: &IdentData) -> u32 { - if let Some(index) = self.idents.get(data) { - return *index; - } - - let index = self.idents.len() as u32; - self.ident_data.push(data.clone()); - self.idents.insert(data.clone(), index); - index - } - - fn get(&self, index: u32) -> &IdentData { - &self.ident_data[index as usize] - } - - #[allow(unused)] - fn get_mut(&mut self, index: u32) -> &mut IdentData { - self.ident_data.get_mut(index as usize).expect("Should be consistent") - } -} - pub struct FreeFunctions; #[derive(Default)] @@ -141,12 +107,13 @@ impl server::TokenStream for RustAnalyzer { Self::TokenStream::from_iter(vec![tree]) } - bridge::TokenTree::Ident(IdentId(index)) => { - let IdentData(ident) = self.ident_interner.get(index).clone(); - let ident: tt::Ident = ident; - let leaf = tt::Leaf::from(ident); - let tree = TokenTree::from(leaf); - Self::TokenStream::from_iter(vec![tree]) + bridge::TokenTree::Ident(symbol) => { + todo!("implement"); + // let IdentData(ident) = self.ident_interner.get(index).clone(); + // let ident: tt::Ident = ident; + // let leaf = tt::Leaf::from(ident); + // let tree = TokenTree::from(leaf); + // Self::TokenStream::from_iter(vec![tree]) } bridge::TokenTree::Literal(literal) => { @@ -210,7 +177,8 @@ impl server::TokenStream for RustAnalyzer { .into_iter() .map(|tree| match tree { tt::TokenTree::Leaf(tt::Leaf::Ident(ident)) => { - bridge::TokenTree::Ident(IdentId(self.ident_interner.intern(&IdentData(ident)))) + todo!("implement"); + // bridge::TokenTree::Ident(Symbol(self.ident_interner.intern(&IdentData(ident)))) } tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) => bridge::TokenTree::Literal(lit), tt::TokenTree::Leaf(tt::Leaf::Punct(punct)) => { @@ -236,162 +204,36 @@ impl server::TokenStream for RustAnalyzer { } } -fn delim_to_internal(d: bridge::Delimiter) -> Option { +fn delim_to_internal(d: proc_macro::Delimiter) -> Option { let kind = match d { - bridge::Delimiter::Parenthesis => tt::DelimiterKind::Parenthesis, - bridge::Delimiter::Brace => tt::DelimiterKind::Brace, - bridge::Delimiter::Bracket => tt::DelimiterKind::Bracket, - bridge::Delimiter::None => return None, + proc_macro::Delimiter::Parenthesis => tt::DelimiterKind::Parenthesis, + proc_macro::Delimiter::Brace => tt::DelimiterKind::Brace, + proc_macro::Delimiter::Bracket => tt::DelimiterKind::Bracket, + proc_macro::Delimiter::None => return None, }; Some(tt::Delimiter { id: tt::TokenId::unspecified(), kind }) } -fn delim_to_external(d: Option) -> bridge::Delimiter { +fn delim_to_external(d: Option) -> proc_macro::Delimiter { match d.map(|it| it.kind) { - Some(tt::DelimiterKind::Parenthesis) => bridge::Delimiter::Parenthesis, - Some(tt::DelimiterKind::Brace) => bridge::Delimiter::Brace, - Some(tt::DelimiterKind::Bracket) => bridge::Delimiter::Bracket, - None => bridge::Delimiter::None, + Some(tt::DelimiterKind::Parenthesis) => proc_macro::Delimiter::Parenthesis, + Some(tt::DelimiterKind::Brace) => proc_macro::Delimiter::Brace, + Some(tt::DelimiterKind::Bracket) => proc_macro::Delimiter::Bracket, + None => proc_macro::Delimiter::None, } } -fn spacing_to_internal(spacing: bridge::Spacing) -> Spacing { +fn spacing_to_internal(spacing: proc_macro::Spacing) -> Spacing { match spacing { - bridge::Spacing::Alone => Spacing::Alone, - bridge::Spacing::Joint => Spacing::Joint, + proc_macro::Spacing::Alone => Spacing::Alone, + proc_macro::Spacing::Joint => Spacing::Joint, } } -fn spacing_to_external(spacing: Spacing) -> bridge::Spacing { +fn spacing_to_external(spacing: Spacing) -> proc_macro::Spacing { match spacing { - Spacing::Alone => bridge::Spacing::Alone, - Spacing::Joint => bridge::Spacing::Joint, - } -} - -impl server::Ident for RustAnalyzer { - fn new(&mut self, string: &str, span: Self::Span, _is_raw: bool) -> Self::Ident { - IdentId(self.ident_interner.intern(&IdentData(tt::Ident { text: string.into(), id: span }))) - } - - fn span(&mut self, ident: Self::Ident) -> Self::Span { - self.ident_interner.get(ident.0).0.id - } - fn with_span(&mut self, ident: Self::Ident, span: Self::Span) -> Self::Ident { - let data = self.ident_interner.get(ident.0); - let new = IdentData(tt::Ident { id: span, ..data.0.clone() }); - IdentId(self.ident_interner.intern(&new)) - } -} - -impl server::Literal for RustAnalyzer { - fn debug_kind(&mut self, _literal: &Self::Literal) -> String { - // r-a: debug_kind and suffix are unsupported; corresponding client code has been changed to not call these. - // They must still be present to be ABI-compatible and work with upstream proc_macro. - "".to_owned() - } - fn from_str(&mut self, s: &str) -> Result { - Ok(Literal { text: s.into(), id: tt::TokenId::unspecified() }) - } - fn symbol(&mut self, literal: &Self::Literal) -> String { - literal.text.to_string() - } - fn suffix(&mut self, _literal: &Self::Literal) -> Option { - None - } - - fn to_string(&mut self, literal: &Self::Literal) -> String { - literal.to_string() - } - - fn integer(&mut self, n: &str) -> Self::Literal { - let n = match n.parse::() { - Ok(n) => n.to_string(), - Err(_) => n.parse::().unwrap().to_string(), - }; - Literal { text: n.into(), id: tt::TokenId::unspecified() } - } - - fn typed_integer(&mut self, n: &str, kind: &str) -> Self::Literal { - macro_rules! def_suffixed_integer { - ($kind:ident, $($ty:ty),*) => { - match $kind { - $( - stringify!($ty) => { - let n: $ty = n.parse().unwrap(); - format!(concat!("{}", stringify!($ty)), n) - } - )* - _ => unimplemented!("unknown args for typed_integer: n {}, kind {}", n, $kind), - } - } - } - - let text = def_suffixed_integer! {kind, u8, u16, u32, u64, u128, usize, i8, i16, i32, i64, i128, isize}; - - Literal { text: text.into(), id: tt::TokenId::unspecified() } - } - - fn float(&mut self, n: &str) -> Self::Literal { - let n: f64 = n.parse().unwrap(); - let mut text = f64::to_string(&n); - if !text.contains('.') { - text += ".0" - } - Literal { text: text.into(), id: tt::TokenId::unspecified() } - } - - fn f32(&mut self, n: &str) -> Self::Literal { - let n: f32 = n.parse().unwrap(); - let text = format!("{}f32", n); - Literal { text: text.into(), id: tt::TokenId::unspecified() } - } - - fn f64(&mut self, n: &str) -> Self::Literal { - let n: f64 = n.parse().unwrap(); - let text = format!("{}f64", n); - Literal { text: text.into(), id: tt::TokenId::unspecified() } - } - - fn string(&mut self, string: &str) -> Self::Literal { - let mut escaped = String::new(); - for ch in string.chars() { - escaped.extend(ch.escape_debug()); - } - Literal { text: format!("\"{}\"", escaped).into(), id: tt::TokenId::unspecified() } - } - - fn character(&mut self, ch: char) -> Self::Literal { - Literal { text: format!("'{}'", ch).into(), id: tt::TokenId::unspecified() } - } - - fn byte_string(&mut self, bytes: &[u8]) -> Self::Literal { - let string = bytes - .iter() - .cloned() - .flat_map(ascii::escape_default) - .map(Into::::into) - .collect::(); - - Literal { text: format!("b\"{}\"", string).into(), id: tt::TokenId::unspecified() } - } - - fn span(&mut self, literal: &Self::Literal) -> Self::Span { - literal.id - } - - fn set_span(&mut self, literal: &mut Self::Literal, span: Self::Span) { - literal.id = span; - } - - fn subspan( - &mut self, - _literal: &Self::Literal, - _start: Bound, - _end: Bound, - ) -> Option { - // FIXME handle span - None + Spacing::Alone => proc_macro::Spacing::Alone, + Spacing::Joint => proc_macro::Spacing::Joint, } } diff --git a/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server/symbol.rs b/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server/symbol.rs new file mode 100644 index 000000000000..b97e2aecf331 --- /dev/null +++ b/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server/symbol.rs @@ -0,0 +1,34 @@ +use std::collections::HashMap; +use tt::SmolStr; + +// Identifier for an interned symbol. +#[derive(Hash, Eq, PartialEq, Copy, Clone)] +pub struct Symbol(u32); + +#[derive(Default)] +struct IdentInterner { + idents: HashMap, + ident_data: Vec, +} + +impl IdentInterner { + fn intern(&mut self, data: &str) -> Symbol { + if let Some(index) = self.idents.get(data) { + return *index; + } + + let index = self.idents.len() as u32; + self.ident_data.push(data.clone()); + self.idents.insert(data.clone(), index); + index + } + + fn get(&self, index: u32) -> &SmolStr { + &self.ident_data[index as usize] + } + + #[allow(unused)] + fn get_mut(&mut self, index: u32) -> &mut SmolStr { + self.ident_data.get_mut(index as usize).expect("Should be consistent") + } +} From 191db9fed4df1ed72fde44bbc8e6f988478db9f5 Mon Sep 17 00:00:00 2001 From: Amos Wenger Date: Wed, 20 Jul 2022 18:43:59 +0200 Subject: [PATCH 04/18] stub missing APIs --- .../src/abis/abi_sysroot/ra_server.rs | 63 ++++++++++++++----- .../src/abis/abi_sysroot/ra_server/symbol.rs | 7 ++- .../abi_sysroot/ra_server/token_stream.rs | 11 +--- 3 files changed, 56 insertions(+), 25 deletions(-) diff --git a/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server.rs b/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server.rs index 9265cd6b6b79..267668783041 100644 --- a/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server.rs +++ b/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server.rs @@ -14,12 +14,13 @@ use super::proc_macro::{ }; mod token_stream; -pub use token_stream::*; +pub use token_stream::TokenStream; +use token_stream::TokenStreamBuilder; mod symbol; pub use symbol::*; -use std::iter::FromIterator; +use std::{iter::FromIterator, ops::Bound}; type Group = tt::Subtree; type TokenTree = tt::TokenTree; @@ -76,6 +77,13 @@ impl server::FreeFunctions for RustAnalyzer { // https://github.com/rust-lang/rust/pull/71858 } fn track_path(&mut self, _path: &str) {} + + fn literal_from_str( + &mut self, + _s: &str, + ) -> Result, ()> { + todo!() + } } impl server::TokenStream for RustAnalyzer { @@ -92,7 +100,7 @@ impl server::TokenStream for RustAnalyzer { } fn from_token_tree( &mut self, - tree: bridge::TokenTree, + tree: bridge::TokenTree, ) -> Self::TokenStream { match tree { bridge::TokenTree::Group(group) => { @@ -107,8 +115,8 @@ impl server::TokenStream for RustAnalyzer { Self::TokenStream::from_iter(vec![tree]) } - bridge::TokenTree::Ident(symbol) => { - todo!("implement"); + bridge::TokenTree::Ident(_symbol) => { + todo!("convert Ident bridge=>TokenStream"); // let IdentData(ident) = self.ident_interner.get(index).clone(); // let ident: tt::Ident = ident; // let leaf = tt::Leaf::from(ident); @@ -116,10 +124,11 @@ impl server::TokenStream for RustAnalyzer { // Self::TokenStream::from_iter(vec![tree]) } - bridge::TokenTree::Literal(literal) => { - let leaf = tt::Leaf::from(literal); - let tree = TokenTree::from(leaf); - Self::TokenStream::from_iter(vec![tree]) + bridge::TokenTree::Literal(_literal) => { + todo!("convert Literal bridge=>TokenStream"); + // let leaf = tt::Leaf::from(literal); + // let tree = TokenTree::from(leaf); + // Self::TokenStream::from_iter(vec![tree]) } bridge::TokenTree::Punct(p) => { @@ -142,7 +151,7 @@ impl server::TokenStream for RustAnalyzer { fn concat_trees( &mut self, base: Option, - trees: Vec>, + trees: Vec>, ) -> Self::TokenStream { let mut builder = TokenStreamBuilder::new(); if let Some(base) = base { @@ -172,15 +181,18 @@ impl server::TokenStream for RustAnalyzer { fn into_trees( &mut self, stream: Self::TokenStream, - ) -> Vec> { + ) -> Vec> { stream .into_iter() .map(|tree| match tree { - tt::TokenTree::Leaf(tt::Leaf::Ident(ident)) => { - todo!("implement"); + tt::TokenTree::Leaf(tt::Leaf::Ident(_ident)) => { + todo!("convert Ident tt=>bridge"); // bridge::TokenTree::Ident(Symbol(self.ident_interner.intern(&IdentData(ident)))) } - tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) => bridge::TokenTree::Literal(lit), + tt::TokenTree::Leaf(tt::Leaf::Literal(_lit)) => { + todo!("convert Literal tt=>bridge"); + // bridge::TokenTree::Literal(lit) + } tt::TokenTree::Leaf(tt::Leaf::Punct(punct)) => { bridge::TokenTree::Punct(bridge::Punct { ch: punct.char as u8, @@ -317,6 +329,15 @@ impl server::Span for RustAnalyzer { // Just return the first span again, because some macros will unwrap the result. Some(first) } + fn subspan( + &mut self, + span: Self::Span, + _start: Bound, + _end: Bound, + ) -> Option { + // Just return the span again, because some macros will unwrap the result. + Some(span) + } fn resolved_at(&mut self, _span: Self::Span, _at: Self::Span) -> Self::Span { // FIXME handle span tt::TokenId::unspecified() @@ -343,6 +364,12 @@ impl server::MultiSpan for RustAnalyzer { } } +impl server::Symbol for RustAnalyzer { + fn normalize_and_validate_ident(&mut self, _string: &str) -> Result { + todo!() + } +} + impl server::Server for RustAnalyzer { fn globals(&mut self) -> bridge::ExpnGlobals { bridge::ExpnGlobals { @@ -351,6 +378,14 @@ impl server::Server for RustAnalyzer { mixed_site: Span::unspecified(), } } + + fn intern_symbol(_ident: &str) -> Self::Symbol { + todo!("intern_symbol") + } + + fn with_symbol_string(_symbol: &Self::Symbol, _f: impl FnOnce(&str)) { + todo!("with_symbol_string") + } } #[cfg(test)] diff --git a/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server/symbol.rs b/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server/symbol.rs index b97e2aecf331..b045f762b880 100644 --- a/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server/symbol.rs +++ b/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server/symbol.rs @@ -14,13 +14,14 @@ struct IdentInterner { impl IdentInterner { fn intern(&mut self, data: &str) -> Symbol { if let Some(index) = self.idents.get(data) { - return *index; + return Symbol(*index); } let index = self.idents.len() as u32; + let data = SmolStr::from(data); self.ident_data.push(data.clone()); - self.idents.insert(data.clone(), index); - index + self.idents.insert(data, index); + Symbol(index) } fn get(&self, index: u32) -> &SmolStr { diff --git a/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server/token_stream.rs b/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server/token_stream.rs index 9a31f2ebf87c..c8bf2ecdd3d9 100644 --- a/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server/token_stream.rs +++ b/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server/token_stream.rs @@ -163,20 +163,15 @@ pub mod token_stream { } impl TokenStreamBuilder { - fn new() -> TokenStreamBuilder { + pub(super) fn new() -> TokenStreamBuilder { TokenStreamBuilder { acc: TokenStream::new() } } - fn push(&mut self, stream: TokenStream) { + pub(super) fn push(&mut self, stream: TokenStream) { self.acc.extend(stream.into_iter()) } - fn build(self) -> TokenStream { + pub(super) fn build(self) -> TokenStream { self.acc } } - -#[derive(Clone)] -pub struct TokenStreamIter { - trees: std::vec::IntoIter, -} From d25b61030e97c2bb95f7dece2c6cf3cf271b6fe6 Mon Sep 17 00:00:00 2001 From: Amos Wenger Date: Wed, 20 Jul 2022 19:13:06 +0200 Subject: [PATCH 05/18] Add literal/ident conversion, tests pass --- Cargo.lock | 5 +- crates/proc-macro-srv/Cargo.toml | 1 + .../src/abis/abi_sysroot/ra_server.rs | 63 ++++++++++++------- .../src/abis/abi_sysroot/ra_server/symbol.rs | 22 +++---- crates/proc-macro-srv/src/abis/mod.rs | 16 +++++ 5 files changed, 71 insertions(+), 36 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a960012c8cff..5d313b930835 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1022,9 +1022,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.12.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7709cef83f0c1f58f666e746a08b21e0085f7440fa6a29cc194d68aac97a4225" +checksum = "18a6dbe30758c9f83eb00cbea4ac95966305f5a7772f3f42ebfc7fc7eddbd8e1" [[package]] name = "oorandom" @@ -1167,6 +1167,7 @@ dependencies = [ "mbe", "memmap2", "object 0.29.0", + "once_cell", "paths", "proc-macro-api", "proc-macro-test", diff --git a/crates/proc-macro-srv/Cargo.toml b/crates/proc-macro-srv/Cargo.toml index e39026ac70bf..06348afdede0 100644 --- a/crates/proc-macro-srv/Cargo.toml +++ b/crates/proc-macro-srv/Cargo.toml @@ -24,6 +24,7 @@ tt = { path = "../tt", version = "0.0.0" } mbe = { path = "../mbe", version = "0.0.0" } paths = { path = "../paths", version = "0.0.0" } proc-macro-api = { path = "../proc-macro-api", version = "0.0.0" } +once_cell = "1.13.0" [dev-dependencies] expect-test = "1.4.0" diff --git a/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server.rs b/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server.rs index 267668783041..55649ab8101d 100644 --- a/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server.rs +++ b/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server.rs @@ -115,20 +115,29 @@ impl server::TokenStream for RustAnalyzer { Self::TokenStream::from_iter(vec![tree]) } - bridge::TokenTree::Ident(_symbol) => { - todo!("convert Ident bridge=>TokenStream"); - // let IdentData(ident) = self.ident_interner.get(index).clone(); - // let ident: tt::Ident = ident; - // let leaf = tt::Leaf::from(ident); - // let tree = TokenTree::from(leaf); - // Self::TokenStream::from_iter(vec![tree]) + bridge::TokenTree::Ident(ident) => { + // FIXME: handle raw idents + let text = SYMBOL_INTERNER.lock().unwrap().get(&ident.sym).clone(); + let ident: tt::Ident = tt::Ident { text, id: ident.span }; + let leaf = tt::Leaf::from(ident); + let tree = TokenTree::from(leaf); + Self::TokenStream::from_iter(vec![tree]) } - bridge::TokenTree::Literal(_literal) => { - todo!("convert Literal bridge=>TokenStream"); - // let leaf = tt::Leaf::from(literal); - // let tree = TokenTree::from(leaf); - // Self::TokenStream::from_iter(vec![tree]) + bridge::TokenTree::Literal(literal) => { + let symbol = SYMBOL_INTERNER.lock().unwrap().get(&literal.symbol).clone(); + + let text: tt::SmolStr = if let Some(suffix) = literal.suffix { + let suffix = SYMBOL_INTERNER.lock().unwrap().get(&suffix).clone(); + format!("{symbol}{suffix}").into() + } else { + symbol + }; + + let literal = tt::Literal { text, id: literal.span }; + let leaf = tt::Leaf::from(literal); + let tree = TokenTree::from(leaf); + Self::TokenStream::from_iter(vec![tree]) } bridge::TokenTree::Punct(p) => { @@ -185,13 +194,23 @@ impl server::TokenStream for RustAnalyzer { stream .into_iter() .map(|tree| match tree { - tt::TokenTree::Leaf(tt::Leaf::Ident(_ident)) => { - todo!("convert Ident tt=>bridge"); - // bridge::TokenTree::Ident(Symbol(self.ident_interner.intern(&IdentData(ident)))) + tt::TokenTree::Leaf(tt::Leaf::Ident(ident)) => { + bridge::TokenTree::Ident(bridge::Ident { + sym: SYMBOL_INTERNER.lock().unwrap().intern(&ident.text), + // FIXME: handle raw idents + is_raw: false, + span: ident.id, + }) } - tt::TokenTree::Leaf(tt::Leaf::Literal(_lit)) => { - todo!("convert Literal tt=>bridge"); - // bridge::TokenTree::Literal(lit) + tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) => { + bridge::TokenTree::Literal(bridge::Literal { + // FIXME: handle literal kinds + kind: bridge::LitKind::Err, + symbol: SYMBOL_INTERNER.lock().unwrap().intern(&lit.text), + // FIXME: handle suffixes + suffix: None, + span: lit.id, + }) } tt::TokenTree::Leaf(tt::Leaf::Punct(punct)) => { bridge::TokenTree::Punct(bridge::Punct { @@ -379,12 +398,12 @@ impl server::Server for RustAnalyzer { } } - fn intern_symbol(_ident: &str) -> Self::Symbol { - todo!("intern_symbol") + fn intern_symbol(ident: &str) -> Self::Symbol { + SYMBOL_INTERNER.lock().unwrap().intern(&tt::SmolStr::from(ident)) } - fn with_symbol_string(_symbol: &Self::Symbol, _f: impl FnOnce(&str)) { - todo!("with_symbol_string") + fn with_symbol_string(symbol: &Self::Symbol, f: impl FnOnce(&str)) { + f(SYMBOL_INTERNER.lock().unwrap().get(symbol).as_str()) } } diff --git a/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server/symbol.rs b/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server/symbol.rs index b045f762b880..5bfc62a301b7 100644 --- a/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server/symbol.rs +++ b/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server/symbol.rs @@ -1,18 +1,21 @@ -use std::collections::HashMap; +use once_cell::sync::Lazy; +use std::{collections::HashMap, sync::Mutex}; use tt::SmolStr; -// Identifier for an interned symbol. +pub(super) static SYMBOL_INTERNER: Lazy> = Lazy::new(|| Default::default()); + +// ID for an interned symbol. #[derive(Hash, Eq, PartialEq, Copy, Clone)] pub struct Symbol(u32); #[derive(Default)] -struct IdentInterner { +pub(super) struct SymbolInterner { idents: HashMap, ident_data: Vec, } -impl IdentInterner { - fn intern(&mut self, data: &str) -> Symbol { +impl SymbolInterner { + pub(super) fn intern(&mut self, data: &str) -> Symbol { if let Some(index) = self.idents.get(data) { return Symbol(*index); } @@ -24,12 +27,7 @@ impl IdentInterner { Symbol(index) } - fn get(&self, index: u32) -> &SmolStr { - &self.ident_data[index as usize] - } - - #[allow(unused)] - fn get_mut(&mut self, index: u32) -> &mut SmolStr { - self.ident_data.get_mut(index as usize).expect("Should be consistent") + pub(super) fn get(&self, index: &Symbol) -> &SmolStr { + &self.ident_data[index.0 as usize] } } diff --git a/crates/proc-macro-srv/src/abis/mod.rs b/crates/proc-macro-srv/src/abis/mod.rs index 62479b7ccd6b..f1083a9284b3 100644 --- a/crates/proc-macro-srv/src/abis/mod.rs +++ b/crates/proc-macro-srv/src/abis/mod.rs @@ -37,6 +37,8 @@ use super::dylib::LoadProcMacroDylibError; pub(crate) use abi_1_58::Abi as Abi_1_58; pub(crate) use abi_1_63::Abi as Abi_1_63; pub(crate) use abi_1_64::Abi as Abi_1_64; +#[cfg(feature = "sysroot-abi")] +pub(crate) use abi_sysroot::Abi as Abi_Sysroot; use libloading::Library; use proc_macro_api::{ProcMacroKind, RustCInfo}; @@ -54,6 +56,8 @@ pub(crate) enum Abi { Abi1_58(Abi_1_58), Abi1_63(Abi_1_63), Abi1_64(Abi_1_64), + #[cfg(feature = "sysroot-abi")] + AbiSysroot(Abi_Sysroot), } impl Abi { @@ -71,6 +75,14 @@ impl Abi { symbol_name: String, info: RustCInfo, ) -> Result { + // Gated behind an env var for now to avoid a change in behavior for + // rustup-installed rust-analyzer + #[cfg(feature = "sysroot-abi")] + if std::env::var("PROC_MACRO_SRV_SYSROOT_ABI").is_ok() { + let inner = unsafe { Abi_Sysroot::from_lib(lib, symbol_name) }?; + return Ok(Abi::AbiSysroot(inner)); + } + // FIXME: this should use exclusive ranges when they're stable // https://github.com/rust-lang/rust/issues/37854 match (info.version.0, info.version.1) { @@ -100,6 +112,8 @@ impl Abi { Self::Abi1_58(abi) => abi.expand(macro_name, macro_body, attributes), Self::Abi1_63(abi) => abi.expand(macro_name, macro_body, attributes), Self::Abi1_64(abi) => abi.expand(macro_name, macro_body, attributes), + #[cfg(feature = "sysroot-abi")] + Self::AbiSysroot(abi) => abi.expand(macro_name, macro_body, attributes), } } @@ -108,6 +122,8 @@ impl Abi { Self::Abi1_58(abi) => abi.list_macros(), Self::Abi1_63(abi) => abi.list_macros(), Self::Abi1_64(abi) => abi.list_macros(), + #[cfg(feature = "sysroot-abi")] + Self::AbiSysroot(abi) => abi.list_macros(), } } } From 480f55533473e1438774ccaac677b10f8a1bd211 Mon Sep 17 00:00:00 2001 From: Amos Wenger Date: Wed, 20 Jul 2022 19:23:25 +0200 Subject: [PATCH 06/18] implement literal_from_str (poorly) --- .../proc-macro-srv/src/abis/abi_sysroot/ra_server.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server.rs b/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server.rs index 55649ab8101d..d46f8dc88add 100644 --- a/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server.rs +++ b/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server.rs @@ -80,9 +80,16 @@ impl server::FreeFunctions for RustAnalyzer { fn literal_from_str( &mut self, - _s: &str, + s: &str, ) -> Result, ()> { - todo!() + // TODO: keep track of LitKind and Suffix + let symbol = SYMBOL_INTERNER.lock().unwrap().intern(s); + Ok(bridge::Literal { + kind: bridge::LitKind::Err, + symbol, + suffix: None, + span: tt::TokenId::unspecified(), + }) } } From fdddd832240efc0f1966c90aab75ad24a5d12fac Mon Sep 17 00:00:00 2001 From: Amos Wenger Date: Thu, 21 Jul 2022 13:13:24 +0200 Subject: [PATCH 07/18] Assert that sysroot ABI version matches exactly Otherwise, fall back to the multi ABI scheme, except in testing, where it becomes a hard error. This should make it possible to use a rustup-provided rust-analyzer with proc macro dylibs compiled by older rustcs, and it'll also catch changes to the format of `rustc --version` or the `.rustc` section that would make them impossible to compare for equality. --- crates/proc-macro-api/src/lib.rs | 2 +- crates/proc-macro-api/src/version.rs | 2 +- crates/proc-macro-srv/build.rs | 25 ++++++++++++++++++ crates/proc-macro-srv/src/abis/mod.rs | 37 +++++++++++++++++++++++---- crates/proc-macro-srv/src/dylib.rs | 5 ++-- 5 files changed, 62 insertions(+), 9 deletions(-) create mode 100644 crates/proc-macro-srv/build.rs diff --git a/crates/proc-macro-api/src/lib.rs b/crates/proc-macro-api/src/lib.rs index dbf2fb37e75b..d7010e825aa9 100644 --- a/crates/proc-macro-api/src/lib.rs +++ b/crates/proc-macro-api/src/lib.rs @@ -26,7 +26,7 @@ use crate::{ process::ProcMacroProcessSrv, }; -pub use version::{read_dylib_info, RustCInfo}; +pub use version::{read_dylib_info, read_version, RustCInfo}; #[derive(Copy, Clone, Eq, PartialEq, Debug, Serialize, Deserialize)] pub enum ProcMacroKind { diff --git a/crates/proc-macro-api/src/version.rs b/crates/proc-macro-api/src/version.rs index 66fe16e94f77..8cef971f3325 100644 --- a/crates/proc-macro-api/src/version.rs +++ b/crates/proc-macro-api/src/version.rs @@ -102,7 +102,7 @@ fn read_section<'a>(dylib_binary: &'a [u8], section_name: &str) -> io::Result<&' /// * [some more bytes that we don't really care but about still there] :-) /// Check this issue for more about the bytes layout: /// -fn read_version(dylib_path: &AbsPath) -> io::Result { +pub fn read_version(dylib_path: &AbsPath) -> io::Result { let dylib_file = File::open(dylib_path)?; let dylib_mmaped = unsafe { Mmap::map(&dylib_file) }?; diff --git a/crates/proc-macro-srv/build.rs b/crates/proc-macro-srv/build.rs new file mode 100644 index 000000000000..e3e4b9d87ba3 --- /dev/null +++ b/crates/proc-macro-srv/build.rs @@ -0,0 +1,25 @@ +use std::{env, fs::File, io::Write, path::PathBuf, process::Command}; + +fn main() { + // Determine rustc version `proc-macro-srv` (and thus the sysroot ABI) is + // build with and make it accessible at runtime for ABI selection. + + let mut path = PathBuf::from(env::var_os("OUT_DIR").unwrap()); + path.push("rustc_version.rs"); + let mut f = File::create(&path).unwrap(); + + let rustc = env::var("RUSTC").expect("proc-macro-srv's build script expects RUSTC to be set"); + let output = Command::new(rustc).arg("--version").output().expect("rustc --version must run"); + let version_string = std::str::from_utf8(&output.stdout[..]) + .expect("rustc --version output must be UTF-8") + .trim(); + + write!( + f, + " + #[allow(dead_code)] + pub(crate) const RUSTC_VERSION_STRING: &str = {version_string:?}; + " + ) + .unwrap(); +} diff --git a/crates/proc-macro-srv/src/abis/mod.rs b/crates/proc-macro-srv/src/abis/mod.rs index f1083a9284b3..a59da0f6b10b 100644 --- a/crates/proc-macro-srv/src/abis/mod.rs +++ b/crates/proc-macro-srv/src/abis/mod.rs @@ -29,6 +29,9 @@ mod abi_1_64; #[cfg(feature = "sysroot-abi")] mod abi_sysroot; +// see `build.rs` +include!(concat!(env!("OUT_DIR"), "/rustc_version.rs")); + // Used by `test/utils.rs` #[cfg(test)] pub(crate) use abi_1_64::TokenStream as TestTokenStream; @@ -74,13 +77,37 @@ impl Abi { lib: &Library, symbol_name: String, info: RustCInfo, + #[cfg_attr(not(feature = "sysroot-abi"), allow(unused_variables))] version_string: String, ) -> Result { - // Gated behind an env var for now to avoid a change in behavior for - // rustup-installed rust-analyzer + // the sysroot ABI relies on `extern proc_macro` with unstable features, + // instead of a snapshot of the proc macro bridge's source code. it's only + // enabled if we have an exact version match. #[cfg(feature = "sysroot-abi")] - if std::env::var("PROC_MACRO_SRV_SYSROOT_ABI").is_ok() { - let inner = unsafe { Abi_Sysroot::from_lib(lib, symbol_name) }?; - return Ok(Abi::AbiSysroot(inner)); + { + if version_string == RUSTC_VERSION_STRING { + let inner = unsafe { Abi_Sysroot::from_lib(lib, symbol_name) }?; + return Ok(Abi::AbiSysroot(inner)); + } + + // if we reached this point, versions didn't match. in testing, we + // want that to panic - this could mean that the format of `rustc + // --version` no longer matches the format of the version string + // stored in the `.rustc` section, and we want to catch that in-tree + // with `x.py test` + #[cfg(test)] + { + let allow_mismatch = std::env::var("PROC_MACRO_SRV_ALLOW_SYSROOT_MISMATCH"); + if let Ok("1") = allow_mismatch.as_deref() { + // only used by rust-analyzer developers, when working on the + // sysroot ABI from the rust-analyzer repository - which should + // only happen pre-subtree. this can be removed later. + } else { + panic!( + "sysroot ABI mismatch: dylib rustc version (read from .rustc section): {:?} != proc-macro-srv version (read from 'rustc --version'): {:?}", + version_string, RUSTC_VERSION_STRING + ); + } + } } // FIXME: this should use exclusive ranges when they're stable diff --git a/crates/proc-macro-srv/src/dylib.rs b/crates/proc-macro-srv/src/dylib.rs index 2b6c070fece3..6439fb2130bc 100644 --- a/crates/proc-macro-srv/src/dylib.rs +++ b/crates/proc-macro-srv/src/dylib.rs @@ -12,7 +12,7 @@ use libloading::Library; use memmap2::Mmap; use object::Object; use paths::AbsPath; -use proc_macro_api::{read_dylib_info, ProcMacroKind}; +use proc_macro_api::{read_dylib_info, read_version, ProcMacroKind}; use super::abis::Abi; @@ -122,9 +122,10 @@ impl ProcMacroLibraryLibloading { invalid_data_err(format!("expected an absolute path, got {}", file.display())) })?; let version_info = read_dylib_info(abs_file)?; + let version_string = read_version(abs_file)?; let lib = load_library(file).map_err(invalid_data_err)?; - let abi = Abi::from_lib(&lib, symbol_name, version_info)?; + let abi = Abi::from_lib(&lib, symbol_name, version_info, version_string)?; Ok(ProcMacroLibraryLibloading { _lib: lib, abi }) } } From bbaf4daca02991f9b6ee90051248034a461bf91a Mon Sep 17 00:00:00 2001 From: Amos Wenger Date: Thu, 21 Jul 2022 13:37:41 +0200 Subject: [PATCH 08/18] Pass tidy checks --- crates/proc-macro-srv/build.rs | 6 +++--- crates/proc-macro-srv/src/abis/abi_sysroot/ra_server.rs | 7 ++++--- .../src/abis/abi_sysroot/ra_server/symbol.rs | 2 ++ .../src/abis/abi_sysroot/ra_server/token_stream.rs | 2 ++ 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/crates/proc-macro-srv/build.rs b/crates/proc-macro-srv/build.rs index e3e4b9d87ba3..a8c732f31541 100644 --- a/crates/proc-macro-srv/build.rs +++ b/crates/proc-macro-srv/build.rs @@ -1,9 +1,9 @@ +//! Determine rustc version `proc-macro-srv` (and thus the sysroot ABI) is +//! build with and make it accessible at runtime for ABI selection. + use std::{env, fs::File, io::Write, path::PathBuf, process::Command}; fn main() { - // Determine rustc version `proc-macro-srv` (and thus the sysroot ABI) is - // build with and make it accessible at runtime for ABI selection. - let mut path = PathBuf::from(env::var_os("OUT_DIR").unwrap()); path.push("rustc_version.rs"); let mut f = File::create(&path).unwrap(); diff --git a/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server.rs b/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server.rs index d46f8dc88add..69f1e13fb06d 100644 --- a/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server.rs +++ b/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server.rs @@ -82,7 +82,7 @@ impl server::FreeFunctions for RustAnalyzer { &mut self, s: &str, ) -> Result, ()> { - // TODO: keep track of LitKind and Suffix + // FIXME: keep track of LitKind and Suffix let symbol = SYMBOL_INTERNER.lock().unwrap().intern(s); Ok(bridge::Literal { kind: bridge::LitKind::Err, @@ -391,8 +391,9 @@ impl server::MultiSpan for RustAnalyzer { } impl server::Symbol for RustAnalyzer { - fn normalize_and_validate_ident(&mut self, _string: &str) -> Result { - todo!() + fn normalize_and_validate_ident(&mut self, string: &str) -> Result { + // FIXME: nfc-normalize and validate idents + Ok(::intern_symbol(string)) } } diff --git a/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server/symbol.rs b/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server/symbol.rs index 5bfc62a301b7..d16c2d224405 100644 --- a/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server/symbol.rs +++ b/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server/symbol.rs @@ -1,3 +1,5 @@ +//! Symbol interner for proc-macro-srv + use once_cell::sync::Lazy; use std::{collections::HashMap, sync::Mutex}; use tt::SmolStr; diff --git a/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server/token_stream.rs b/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server/token_stream.rs index c8bf2ecdd3d9..113bb52c1af5 100644 --- a/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server/token_stream.rs +++ b/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server/token_stream.rs @@ -1,3 +1,5 @@ +//! TokenStream implementation used by sysroot ABI + use tt::TokenTree; #[derive(Debug, Default, Clone)] From 30769598a406a30f05d4b8a72430824708bdc73f Mon Sep 17 00:00:00 2001 From: Amos Wenger Date: Thu, 21 Jul 2022 13:57:36 +0200 Subject: [PATCH 09/18] Move version string to RustcInfo, read '.rustc' section only once --- crates/proc-macro-api/src/version.rs | 4 +++- crates/proc-macro-srv/src/abis/mod.rs | 5 ++--- crates/proc-macro-srv/src/dylib.rs | 5 ++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/crates/proc-macro-api/src/version.rs b/crates/proc-macro-api/src/version.rs index 8cef971f3325..030531b80d7b 100644 --- a/crates/proc-macro-api/src/version.rs +++ b/crates/proc-macro-api/src/version.rs @@ -16,6 +16,8 @@ pub struct RustCInfo { pub channel: String, pub commit: Option, pub date: Option, + // something like "rustc 1.58.1 (db9d1b20b 2022-01-20)" + pub version_string: String, } /// Read rustc dylib information @@ -68,7 +70,7 @@ pub fn read_dylib_info(dylib_path: &AbsPath) -> io::Result { } let version = (version_numbers[0], version_numbers[1], version_numbers[2]); - Ok(RustCInfo { version, channel, commit, date }) + Ok(RustCInfo { version, channel, commit, date, version_string: ver_str }) } /// This is used inside read_version() to locate the ".rustc" section diff --git a/crates/proc-macro-srv/src/abis/mod.rs b/crates/proc-macro-srv/src/abis/mod.rs index a59da0f6b10b..bcf3f1184cf6 100644 --- a/crates/proc-macro-srv/src/abis/mod.rs +++ b/crates/proc-macro-srv/src/abis/mod.rs @@ -77,14 +77,13 @@ impl Abi { lib: &Library, symbol_name: String, info: RustCInfo, - #[cfg_attr(not(feature = "sysroot-abi"), allow(unused_variables))] version_string: String, ) -> Result { // the sysroot ABI relies on `extern proc_macro` with unstable features, // instead of a snapshot of the proc macro bridge's source code. it's only // enabled if we have an exact version match. #[cfg(feature = "sysroot-abi")] { - if version_string == RUSTC_VERSION_STRING { + if info.version_string == RUSTC_VERSION_STRING { let inner = unsafe { Abi_Sysroot::from_lib(lib, symbol_name) }?; return Ok(Abi::AbiSysroot(inner)); } @@ -104,7 +103,7 @@ impl Abi { } else { panic!( "sysroot ABI mismatch: dylib rustc version (read from .rustc section): {:?} != proc-macro-srv version (read from 'rustc --version'): {:?}", - version_string, RUSTC_VERSION_STRING + info.version_string, RUSTC_VERSION_STRING ); } } diff --git a/crates/proc-macro-srv/src/dylib.rs b/crates/proc-macro-srv/src/dylib.rs index 6439fb2130bc..2b6c070fece3 100644 --- a/crates/proc-macro-srv/src/dylib.rs +++ b/crates/proc-macro-srv/src/dylib.rs @@ -12,7 +12,7 @@ use libloading::Library; use memmap2::Mmap; use object::Object; use paths::AbsPath; -use proc_macro_api::{read_dylib_info, read_version, ProcMacroKind}; +use proc_macro_api::{read_dylib_info, ProcMacroKind}; use super::abis::Abi; @@ -122,10 +122,9 @@ impl ProcMacroLibraryLibloading { invalid_data_err(format!("expected an absolute path, got {}", file.display())) })?; let version_info = read_dylib_info(abs_file)?; - let version_string = read_version(abs_file)?; let lib = load_library(file).map_err(invalid_data_err)?; - let abi = Abi::from_lib(&lib, symbol_name, version_info, version_string)?; + let abi = Abi::from_lib(&lib, symbol_name, version_info)?; Ok(ProcMacroLibraryLibloading { _lib: lib, abi }) } } From 05d8f5fee7ceacb68e18a6c9c7fc4d15adc75b22 Mon Sep 17 00:00:00 2001 From: Amos Wenger Date: Thu, 21 Jul 2022 14:57:09 +0200 Subject: [PATCH 10/18] Use a thread-local for the symbol interner (1/2) --- Cargo.lock | 1 - crates/proc-macro-srv/Cargo.toml | 1 - .../src/abis/abi_sysroot/ra_server.rs | 17 +++++----- .../src/abis/abi_sysroot/ra_server/symbol.rs | 31 ++++++++++++++----- 4 files changed, 33 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5d313b930835..0204617d5dfd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1167,7 +1167,6 @@ dependencies = [ "mbe", "memmap2", "object 0.29.0", - "once_cell", "paths", "proc-macro-api", "proc-macro-test", diff --git a/crates/proc-macro-srv/Cargo.toml b/crates/proc-macro-srv/Cargo.toml index 06348afdede0..e39026ac70bf 100644 --- a/crates/proc-macro-srv/Cargo.toml +++ b/crates/proc-macro-srv/Cargo.toml @@ -24,7 +24,6 @@ tt = { path = "../tt", version = "0.0.0" } mbe = { path = "../mbe", version = "0.0.0" } paths = { path = "../paths", version = "0.0.0" } proc-macro-api = { path = "../proc-macro-api", version = "0.0.0" } -once_cell = "1.13.0" [dev-dependencies] expect-test = "1.4.0" diff --git a/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server.rs b/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server.rs index 69f1e13fb06d..066e0e26ac87 100644 --- a/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server.rs +++ b/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server.rs @@ -83,7 +83,7 @@ impl server::FreeFunctions for RustAnalyzer { s: &str, ) -> Result, ()> { // FIXME: keep track of LitKind and Suffix - let symbol = SYMBOL_INTERNER.lock().unwrap().intern(s); + let symbol = ThreadLocalSymbolInterner::intern(s); Ok(bridge::Literal { kind: bridge::LitKind::Err, symbol, @@ -124,7 +124,7 @@ impl server::TokenStream for RustAnalyzer { bridge::TokenTree::Ident(ident) => { // FIXME: handle raw idents - let text = SYMBOL_INTERNER.lock().unwrap().get(&ident.sym).clone(); + let text = ThreadLocalSymbolInterner::get_cloned(&ident.sym); let ident: tt::Ident = tt::Ident { text, id: ident.span }; let leaf = tt::Leaf::from(ident); let tree = TokenTree::from(leaf); @@ -132,10 +132,11 @@ impl server::TokenStream for RustAnalyzer { } bridge::TokenTree::Literal(literal) => { - let symbol = SYMBOL_INTERNER.lock().unwrap().get(&literal.symbol).clone(); + // FIXME: remove unnecessary clones here + let symbol = ThreadLocalSymbolInterner::get_cloned(&literal.symbol); let text: tt::SmolStr = if let Some(suffix) = literal.suffix { - let suffix = SYMBOL_INTERNER.lock().unwrap().get(&suffix).clone(); + let suffix = ThreadLocalSymbolInterner::get_cloned(&suffix); format!("{symbol}{suffix}").into() } else { symbol @@ -203,7 +204,7 @@ impl server::TokenStream for RustAnalyzer { .map(|tree| match tree { tt::TokenTree::Leaf(tt::Leaf::Ident(ident)) => { bridge::TokenTree::Ident(bridge::Ident { - sym: SYMBOL_INTERNER.lock().unwrap().intern(&ident.text), + sym: ThreadLocalSymbolInterner::intern(&ident.text), // FIXME: handle raw idents is_raw: false, span: ident.id, @@ -213,7 +214,7 @@ impl server::TokenStream for RustAnalyzer { bridge::TokenTree::Literal(bridge::Literal { // FIXME: handle literal kinds kind: bridge::LitKind::Err, - symbol: SYMBOL_INTERNER.lock().unwrap().intern(&lit.text), + symbol: ThreadLocalSymbolInterner::intern(&lit.text), // FIXME: handle suffixes suffix: None, span: lit.id, @@ -407,11 +408,11 @@ impl server::Server for RustAnalyzer { } fn intern_symbol(ident: &str) -> Self::Symbol { - SYMBOL_INTERNER.lock().unwrap().intern(&tt::SmolStr::from(ident)) + ThreadLocalSymbolInterner::intern(&tt::SmolStr::from(ident)) } fn with_symbol_string(symbol: &Self::Symbol, f: impl FnOnce(&str)) { - f(SYMBOL_INTERNER.lock().unwrap().get(symbol).as_str()) + ThreadLocalSymbolInterner::with(symbol, |s| f(s.as_str())) } } diff --git a/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server/symbol.rs b/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server/symbol.rs index d16c2d224405..294ef76c24aa 100644 --- a/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server/symbol.rs +++ b/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server/symbol.rs @@ -1,23 +1,24 @@ //! Symbol interner for proc-macro-srv -use once_cell::sync::Lazy; -use std::{collections::HashMap, sync::Mutex}; +use std::{cell::RefCell, collections::HashMap}; use tt::SmolStr; -pub(super) static SYMBOL_INTERNER: Lazy> = Lazy::new(|| Default::default()); +thread_local! { + static SYMBOL_INTERNER: RefCell = Default::default(); +} // ID for an interned symbol. #[derive(Hash, Eq, PartialEq, Copy, Clone)] pub struct Symbol(u32); #[derive(Default)] -pub(super) struct SymbolInterner { +struct SymbolInterner { idents: HashMap, ident_data: Vec, } impl SymbolInterner { - pub(super) fn intern(&mut self, data: &str) -> Symbol { + fn intern(&mut self, data: &str) -> Symbol { if let Some(index) = self.idents.get(data) { return Symbol(*index); } @@ -29,7 +30,23 @@ impl SymbolInterner { Symbol(index) } - pub(super) fn get(&self, index: &Symbol) -> &SmolStr { - &self.ident_data[index.0 as usize] + fn get(&self, sym: &Symbol) -> &SmolStr { + &self.ident_data[sym.0 as usize] + } +} + +pub(super) struct ThreadLocalSymbolInterner; + +impl ThreadLocalSymbolInterner { + pub(super) fn intern(data: &str) -> Symbol { + SYMBOL_INTERNER.with(|i| i.borrow_mut().intern(data)) + } + + pub(super) fn with(sym: &Symbol, f: impl FnOnce(&SmolStr) -> T) -> T { + SYMBOL_INTERNER.with(|i| f(i.borrow().get(sym))) + } + + pub(super) fn get_cloned(sym: &Symbol) -> SmolStr { + Self::with(sym, |s| s.clone()) } } From 32ee097580fe473bcdbae5f422c93e99dec348a3 Mon Sep 17 00:00:00 2001 From: Amos Wenger Date: Thu, 21 Jul 2022 18:11:50 +0200 Subject: [PATCH 11/18] Run proc macro expansion in a separate thread (for the thread-local interner) --- Cargo.lock | 25 +++++++++++++++++++++++++ crates/proc-macro-srv/Cargo.toml | 1 + crates/proc-macro-srv/src/lib.rs | 13 ++++++++++--- 3 files changed, 36 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0204617d5dfd..67bfbf010960 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -247,6 +247,20 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ae5588f6b3c3cb05239e90bd110f257254aecd01e4635400391aeae07497845" +dependencies = [ + "cfg-if", + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-epoch", + "crossbeam-queue", + "crossbeam-utils", +] + [[package]] name = "crossbeam-channel" version = "0.5.5" @@ -282,6 +296,16 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "crossbeam-queue" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f25d8400f4a7a5778f0e4e52384a48cbd9b5c495d110786187fc750075277a2" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.10" @@ -1162,6 +1186,7 @@ dependencies = [ name = "proc-macro-srv" version = "0.0.0" dependencies = [ + "crossbeam", "expect-test", "libloading", "mbe", diff --git a/crates/proc-macro-srv/Cargo.toml b/crates/proc-macro-srv/Cargo.toml index e39026ac70bf..5746eac0b379 100644 --- a/crates/proc-macro-srv/Cargo.toml +++ b/crates/proc-macro-srv/Cargo.toml @@ -24,6 +24,7 @@ tt = { path = "../tt", version = "0.0.0" } mbe = { path = "../mbe", version = "0.0.0" } paths = { path = "../paths", version = "0.0.0" } proc-macro-api = { path = "../proc-macro-api", version = "0.0.0" } +crossbeam = "0.8.1" [dev-dependencies] expect-test = "1.4.0" diff --git a/crates/proc-macro-srv/src/lib.rs b/crates/proc-macro-srv/src/lib.rs index 2ab6eba6f424..98a79f5be330 100644 --- a/crates/proc-macro-srv/src/lib.rs +++ b/crates/proc-macro-srv/src/lib.rs @@ -63,9 +63,16 @@ impl ProcMacroSrv { let macro_body = task.macro_body.to_subtree(); let attributes = task.attributes.map(|it| it.to_subtree()); - let result = expander - .expand(&task.macro_name, ¯o_body, attributes.as_ref()) - .map(|it| FlatTree::new(&it)); + let result = crossbeam::scope(|s| { + s.spawn(|_| { + expander + .expand(&task.macro_name, ¯o_body, attributes.as_ref()) + .map(|it| FlatTree::new(&it)) + }) + .join() + .unwrap() + }) + .unwrap(); prev_env.rollback(); From 36d825fd5d01a9796adb55e0ae7fac7ed31dbe9c Mon Sep 17 00:00:00 2001 From: Amos Wenger Date: Thu, 21 Jul 2022 18:29:28 +0200 Subject: [PATCH 12/18] Add test for literals created client-side --- crates/proc-macro-srv/src/tests/mod.rs | 30 ++++++++++++++++++++++++++ crates/proc-macro-test/imp/src/lib.rs | 16 ++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/crates/proc-macro-srv/src/tests/mod.rs b/crates/proc-macro-srv/src/tests/mod.rs index d4be992465ca..30e1ea334359 100644 --- a/crates/proc-macro-srv/src/tests/mod.rs +++ b/crates/proc-macro-srv/src/tests/mod.rs @@ -56,6 +56,35 @@ fn test_fn_like_macro_clone_ident_subtree() { ); } +#[test] +fn test_fn_like_macro_clone_raw_ident() { + assert_expand( + "fn_like_clone_tokens", + "r#\"ident\"#", + expect![[r##" + SUBTREE $ + LITERAL r#"ident"# 4294967295"##]], + ); +} + +#[test] +fn test_fn_like_mk_literals() { + assert_expand( + "fn_like_mk_literals", + r#""#, + expect![[r#" + SUBTREE $ + LITERAL b"byte_string" 4294967295 + LITERAL 'c' 4294967295 + LITERAL "string" 4294967295 + LITERAL "maybe \"raw\"?" 4294967295 + LITERAL 3.14f64 4294967295 + LITERAL 3.14 4294967295 + LITERAL 123i64 4294967295 + LITERAL 123 4294967295"#]], + ); +} + #[test] fn test_fn_like_macro_clone_literals() { assert_expand( @@ -105,6 +134,7 @@ fn list_test_macros() { fn_like_panic [FuncLike] fn_like_error [FuncLike] fn_like_clone_tokens [FuncLike] + fn_like_mk_literals [FuncLike] attr_noop [Attr] attr_panic [Attr] attr_error [Attr] diff --git a/crates/proc-macro-test/imp/src/lib.rs b/crates/proc-macro-test/imp/src/lib.rs index 0082eb7bdaf7..d18b357a040b 100644 --- a/crates/proc-macro-test/imp/src/lib.rs +++ b/crates/proc-macro-test/imp/src/lib.rs @@ -24,6 +24,22 @@ pub fn fn_like_clone_tokens(args: TokenStream) -> TokenStream { clone_stream(args) } +#[proc_macro] +pub fn fn_like_mk_literals(_args: TokenStream) -> TokenStream { + let trees: Vec = vec![ + TokenTree::from(Literal::byte_string(b"byte_string")), + TokenTree::from(Literal::character('c')), + TokenTree::from(Literal::string("string")), + // as of 2022-07-21, there's no method on `Literal` to build a raw + // string or a raw byte string + TokenTree::from(Literal::f64_suffixed(3.14)), + TokenTree::from(Literal::f64_unsuffixed(3.14)), + TokenTree::from(Literal::i64_suffixed(123)), + TokenTree::from(Literal::i64_unsuffixed(123)), + ]; + TokenStream::from_iter(trees) +} + #[proc_macro_attribute] pub fn attr_noop(_args: TokenStream, item: TokenStream) -> TokenStream { item From 9cf99a9c71348c1ed53b6dda7da89d5ad0774b08 Mon Sep 17 00:00:00 2001 From: Amos Wenger Date: Thu, 21 Jul 2022 18:44:41 +0200 Subject: [PATCH 13/18] Stringify literals create client-side properly --- .../src/abis/abi_sysroot/ra_server.rs | 61 ++++++++++++++++--- crates/proc-macro-srv/src/tests/mod.rs | 1 - 2 files changed, 51 insertions(+), 11 deletions(-) diff --git a/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server.rs b/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server.rs index 066e0e26ac87..2e5751756a71 100644 --- a/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server.rs +++ b/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server.rs @@ -132,17 +132,11 @@ impl server::TokenStream for RustAnalyzer { } bridge::TokenTree::Literal(literal) => { - // FIXME: remove unnecessary clones here - let symbol = ThreadLocalSymbolInterner::get_cloned(&literal.symbol); - - let text: tt::SmolStr = if let Some(suffix) = literal.suffix { - let suffix = ThreadLocalSymbolInterner::get_cloned(&suffix); - format!("{symbol}{suffix}").into() - } else { - symbol - }; + let literal = LiteralFormatter(literal); + let text = literal + .with_stringify_parts(|parts| tt::SmolStr::from_iter(parts.iter().copied())); - let literal = tt::Literal { text, id: literal.span }; + let literal = tt::Literal { text, id: literal.0.span }; let leaf = tt::Leaf::from(literal); let tree = TokenTree::from(leaf); Self::TokenStream::from_iter(vec![tree]) @@ -416,6 +410,53 @@ impl server::Server for RustAnalyzer { } } +struct LiteralFormatter(bridge::Literal); + +impl LiteralFormatter { + /// Invokes the callback with a `&[&str]` consisting of each part of the + /// literal's representation. This is done to allow the `ToString` and + /// `Display` implementations to borrow references to symbol values, and + /// both be optimized to reduce overhead. + fn with_stringify_parts(&self, f: impl FnOnce(&[&str]) -> R) -> R { + /// Returns a string containing exactly `num` '#' characters. + /// Uses a 256-character source string literal which is always safe to + /// index with a `u8` index. + fn get_hashes_str(num: u8) -> &'static str { + const HASHES: &str = "\ + ################################################################\ + ################################################################\ + ################################################################\ + ################################################################\ + "; + const _: () = assert!(HASHES.len() == 256); + &HASHES[..num as usize] + } + + self.with_symbol_and_suffix(|symbol, suffix| match self.0.kind { + bridge::LitKind::Byte => f(&["b'", symbol, "'", suffix]), + bridge::LitKind::Char => f(&["'", symbol, "'", suffix]), + bridge::LitKind::Str => f(&["\"", symbol, "\"", suffix]), + bridge::LitKind::StrRaw(n) => { + let hashes = get_hashes_str(n); + f(&["r", hashes, "\"", symbol, "\"", hashes, suffix]) + } + bridge::LitKind::ByteStr => f(&["b\"", symbol, "\"", suffix]), + bridge::LitKind::ByteStrRaw(n) => { + let hashes = get_hashes_str(n); + f(&["br", hashes, "\"", symbol, "\"", hashes, suffix]) + } + _ => f(&[symbol, suffix]), + }) + } + + fn with_symbol_and_suffix(&self, f: impl FnOnce(&str, &str) -> R) -> R { + ThreadLocalSymbolInterner::with(&self.0.symbol, |symbol| match self.0.suffix.as_ref() { + Some(suffix) => ThreadLocalSymbolInterner::with(suffix, |suffix| f(symbol, suffix)), + None => f(symbol, ""), + }) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/proc-macro-srv/src/tests/mod.rs b/crates/proc-macro-srv/src/tests/mod.rs index 30e1ea334359..94c27fad30bf 100644 --- a/crates/proc-macro-srv/src/tests/mod.rs +++ b/crates/proc-macro-srv/src/tests/mod.rs @@ -77,7 +77,6 @@ fn test_fn_like_mk_literals() { LITERAL b"byte_string" 4294967295 LITERAL 'c' 4294967295 LITERAL "string" 4294967295 - LITERAL "maybe \"raw\"?" 4294967295 LITERAL 3.14f64 4294967295 LITERAL 3.14 4294967295 LITERAL 123i64 4294967295 From 941416a1d6a07141dd25eaf608d8fa47ad8a4dec Mon Sep 17 00:00:00 2001 From: Amos Wenger Date: Thu, 21 Jul 2022 18:48:19 +0200 Subject: [PATCH 14/18] Add test for idents incl. raw idents --- crates/proc-macro-srv/src/tests/mod.rs | 15 +++++++++++++++ crates/proc-macro-test/imp/src/lib.rs | 11 ++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/crates/proc-macro-srv/src/tests/mod.rs b/crates/proc-macro-srv/src/tests/mod.rs index 94c27fad30bf..1277106d71d9 100644 --- a/crates/proc-macro-srv/src/tests/mod.rs +++ b/crates/proc-macro-srv/src/tests/mod.rs @@ -84,6 +84,20 @@ fn test_fn_like_mk_literals() { ); } +#[test] +fn test_fn_like_mk_idents() { + // FIXME: this test is wrong: raw should be 'r#raw' but ABIs 1.64 and below + // simply ignore `is_raw` when implementing the `Ident` interface. + assert_expand( + "fn_like_mk_idents", + r#""#, + expect![[r#" + SUBTREE $ + IDENT standard 4294967295 + IDENT raw 4294967295"#]], + ); +} + #[test] fn test_fn_like_macro_clone_literals() { assert_expand( @@ -134,6 +148,7 @@ fn list_test_macros() { fn_like_error [FuncLike] fn_like_clone_tokens [FuncLike] fn_like_mk_literals [FuncLike] + fn_like_mk_idents [FuncLike] attr_noop [Attr] attr_panic [Attr] attr_error [Attr] diff --git a/crates/proc-macro-test/imp/src/lib.rs b/crates/proc-macro-test/imp/src/lib.rs index d18b357a040b..7760774a3fc3 100644 --- a/crates/proc-macro-test/imp/src/lib.rs +++ b/crates/proc-macro-test/imp/src/lib.rs @@ -2,7 +2,7 @@ #![warn(rust_2018_idioms, unused_lifetimes, semicolon_in_expressions_from_macros)] -use proc_macro::{Group, Ident, Literal, Punct, TokenStream, TokenTree}; +use proc_macro::{Group, Ident, Literal, Punct, Span, TokenStream, TokenTree}; #[proc_macro] pub fn fn_like_noop(args: TokenStream) -> TokenStream { @@ -40,6 +40,15 @@ pub fn fn_like_mk_literals(_args: TokenStream) -> TokenStream { TokenStream::from_iter(trees) } +#[proc_macro] +pub fn fn_like_mk_idents(_args: TokenStream) -> TokenStream { + let trees: Vec = vec![ + TokenTree::from(Ident::new("standard", Span::call_site())), + TokenTree::from(Ident::new_raw("raw", Span::call_site())), + ]; + TokenStream::from_iter(trees) +} + #[proc_macro_attribute] pub fn attr_noop(_args: TokenStream, item: TokenStream) -> TokenStream { item From 246947b7791fd470e8e49c08d30c4c740eb62bc5 Mon Sep 17 00:00:00 2001 From: Amos Wenger Date: Thu, 21 Jul 2022 19:13:44 +0200 Subject: [PATCH 15/18] Fix raw ident handling (a little) --- crates/proc-macro-srv/src/tests/mod.rs | 6 +++--- crates/proc-macro-test/imp/src/lib.rs | 9 ++++++++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/crates/proc-macro-srv/src/tests/mod.rs b/crates/proc-macro-srv/src/tests/mod.rs index 1277106d71d9..07222907f088 100644 --- a/crates/proc-macro-srv/src/tests/mod.rs +++ b/crates/proc-macro-srv/src/tests/mod.rs @@ -60,10 +60,10 @@ fn test_fn_like_macro_clone_ident_subtree() { fn test_fn_like_macro_clone_raw_ident() { assert_expand( "fn_like_clone_tokens", - "r#\"ident\"#", - expect![[r##" + "r#async", + expect![[r#" SUBTREE $ - LITERAL r#"ident"# 4294967295"##]], + IDENT async 4294967295"#]], ); } diff --git a/crates/proc-macro-test/imp/src/lib.rs b/crates/proc-macro-test/imp/src/lib.rs index 7760774a3fc3..feeacdb6407a 100644 --- a/crates/proc-macro-test/imp/src/lib.rs +++ b/crates/proc-macro-test/imp/src/lib.rs @@ -90,7 +90,14 @@ fn clone_tree(t: TokenTree) -> TokenTree { new.set_span(orig.span()); TokenTree::Group(new) } - TokenTree::Ident(orig) => TokenTree::Ident(Ident::new(&orig.to_string(), orig.span())), + TokenTree::Ident(orig) => { + let s = orig.to_string(); + if let Some(rest) = s.strip_prefix("r#") { + TokenTree::Ident(Ident::new_raw(rest, orig.span())) + } else { + TokenTree::Ident(Ident::new(&s, orig.span())) + } + } TokenTree::Punct(orig) => { let mut new = Punct::new(orig.as_char(), orig.spacing()); new.set_span(orig.span()); From 48bcc229bf80bfc7e3db2d4728fd83381d9e2ce6 Mon Sep 17 00:00:00 2001 From: Amos Wenger Date: Thu, 21 Jul 2022 21:32:46 +0200 Subject: [PATCH 16/18] Move interner methods to Symbol, return SmolStr directly since it's ref-counted --- .../src/abis/abi_sysroot/ra_server.rs | 20 +++++++------- .../src/abis/abi_sysroot/ra_server/symbol.rs | 26 +++++++------------ 2 files changed, 19 insertions(+), 27 deletions(-) diff --git a/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server.rs b/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server.rs index 2e5751756a71..46882845a807 100644 --- a/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server.rs +++ b/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server.rs @@ -83,10 +83,9 @@ impl server::FreeFunctions for RustAnalyzer { s: &str, ) -> Result, ()> { // FIXME: keep track of LitKind and Suffix - let symbol = ThreadLocalSymbolInterner::intern(s); Ok(bridge::Literal { kind: bridge::LitKind::Err, - symbol, + symbol: Symbol::intern(s), suffix: None, span: tt::TokenId::unspecified(), }) @@ -124,7 +123,7 @@ impl server::TokenStream for RustAnalyzer { bridge::TokenTree::Ident(ident) => { // FIXME: handle raw idents - let text = ThreadLocalSymbolInterner::get_cloned(&ident.sym); + let text = ident.sym.text(); let ident: tt::Ident = tt::Ident { text, id: ident.span }; let leaf = tt::Leaf::from(ident); let tree = TokenTree::from(leaf); @@ -198,7 +197,7 @@ impl server::TokenStream for RustAnalyzer { .map(|tree| match tree { tt::TokenTree::Leaf(tt::Leaf::Ident(ident)) => { bridge::TokenTree::Ident(bridge::Ident { - sym: ThreadLocalSymbolInterner::intern(&ident.text), + sym: Symbol::intern(&ident.text), // FIXME: handle raw idents is_raw: false, span: ident.id, @@ -208,7 +207,7 @@ impl server::TokenStream for RustAnalyzer { bridge::TokenTree::Literal(bridge::Literal { // FIXME: handle literal kinds kind: bridge::LitKind::Err, - symbol: ThreadLocalSymbolInterner::intern(&lit.text), + symbol: Symbol::intern(&lit.text), // FIXME: handle suffixes suffix: None, span: lit.id, @@ -402,11 +401,11 @@ impl server::Server for RustAnalyzer { } fn intern_symbol(ident: &str) -> Self::Symbol { - ThreadLocalSymbolInterner::intern(&tt::SmolStr::from(ident)) + Symbol::intern(&tt::SmolStr::from(ident)) } fn with_symbol_string(symbol: &Self::Symbol, f: impl FnOnce(&str)) { - ThreadLocalSymbolInterner::with(symbol, |s| f(s.as_str())) + f(symbol.text().as_str()) } } @@ -450,10 +449,9 @@ impl LiteralFormatter { } fn with_symbol_and_suffix(&self, f: impl FnOnce(&str, &str) -> R) -> R { - ThreadLocalSymbolInterner::with(&self.0.symbol, |symbol| match self.0.suffix.as_ref() { - Some(suffix) => ThreadLocalSymbolInterner::with(suffix, |suffix| f(symbol, suffix)), - None => f(symbol, ""), - }) + let symbol = self.0.symbol.text(); + let suffix = self.0.suffix.map(|s| s.text()).unwrap_or_default(); + f(symbol.as_str(), suffix.as_str()) } } diff --git a/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server/symbol.rs b/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server/symbol.rs index 294ef76c24aa..51dfba2ea9fb 100644 --- a/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server/symbol.rs +++ b/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server/symbol.rs @@ -11,6 +11,16 @@ thread_local! { #[derive(Hash, Eq, PartialEq, Copy, Clone)] pub struct Symbol(u32); +impl Symbol { + pub fn intern(data: &str) -> Symbol { + SYMBOL_INTERNER.with(|i| i.borrow_mut().intern(data)) + } + + pub fn text(&self) -> SmolStr { + SYMBOL_INTERNER.with(|i| i.borrow().get(self).clone()) + } +} + #[derive(Default)] struct SymbolInterner { idents: HashMap, @@ -34,19 +44,3 @@ impl SymbolInterner { &self.ident_data[sym.0 as usize] } } - -pub(super) struct ThreadLocalSymbolInterner; - -impl ThreadLocalSymbolInterner { - pub(super) fn intern(data: &str) -> Symbol { - SYMBOL_INTERNER.with(|i| i.borrow_mut().intern(data)) - } - - pub(super) fn with(sym: &Symbol, f: impl FnOnce(&SmolStr) -> T) -> T { - SYMBOL_INTERNER.with(|i| f(i.borrow().get(sym))) - } - - pub(super) fn get_cloned(sym: &Symbol) -> SmolStr { - Self::with(sym, |s| s.clone()) - } -} From 39db9cdb7dc7310b6488c0c904ecae71765649cd Mon Sep 17 00:00:00 2001 From: Amos Wenger Date: Thu, 21 Jul 2022 21:35:15 +0200 Subject: [PATCH 17/18] Use std::panic::resume_unwind --- crates/proc-macro-srv/src/lib.rs | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/crates/proc-macro-srv/src/lib.rs b/crates/proc-macro-srv/src/lib.rs index 98a79f5be330..5b96b5c69296 100644 --- a/crates/proc-macro-srv/src/lib.rs +++ b/crates/proc-macro-srv/src/lib.rs @@ -64,15 +64,23 @@ impl ProcMacroSrv { let macro_body = task.macro_body.to_subtree(); let attributes = task.attributes.map(|it| it.to_subtree()); let result = crossbeam::scope(|s| { - s.spawn(|_| { - expander - .expand(&task.macro_name, ¯o_body, attributes.as_ref()) - .map(|it| FlatTree::new(&it)) - }) - .join() - .unwrap() - }) - .unwrap(); + let res = s + .spawn(|_| { + expander + .expand(&task.macro_name, ¯o_body, attributes.as_ref()) + .map(|it| FlatTree::new(&it)) + }) + .join(); + + match res { + Ok(res) => res, + Err(e) => std::panic::resume_unwind(e), + } + }); + let result = match result { + Ok(result) => result, + Err(e) => std::panic::resume_unwind(e), + }; prev_env.rollback(); From e591ff32693b77ad3caf3e5b6d0301f55a6aa543 Mon Sep 17 00:00:00 2001 From: Amos Wenger Date: Fri, 22 Jul 2022 14:23:36 +0200 Subject: [PATCH 18/18] Add comment about replacing crossbeam with std scoped threads when they land --- crates/proc-macro-srv/src/lib.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/proc-macro-srv/src/lib.rs b/crates/proc-macro-srv/src/lib.rs index 5b96b5c69296..4b1858b8ed89 100644 --- a/crates/proc-macro-srv/src/lib.rs +++ b/crates/proc-macro-srv/src/lib.rs @@ -63,6 +63,8 @@ impl ProcMacroSrv { let macro_body = task.macro_body.to_subtree(); let attributes = task.attributes.map(|it| it.to_subtree()); + // FIXME: replace this with std's scoped threads once they stabilize + // (then remove dependency on crossbeam) let result = crossbeam::scope(|s| { let res = s .spawn(|_| {