From 85e558ea6c64380bcfe70384bc13bd1addb92b31 Mon Sep 17 00:00:00 2001 From: l3ops Date: Wed, 16 Feb 2022 10:29:22 +0100 Subject: [PATCH] cleanup formatter codegen + add git integration --- Cargo.lock | 100 +++++++- xtask/codegen/Cargo.toml | 5 +- xtask/codegen/src/formatter.rs | 404 ++++++++++++++++----------------- 3 files changed, 296 insertions(+), 213 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7df2d84ecd9..55616cde802 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -173,6 +173,9 @@ name = "cc" version = "1.0.72" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22a9137b95ea06864e018375b72adfb7db6e6f68cfc8df5a04d00288050485ee" +dependencies = [ + "jobserver", +] [[package]] name = "cfg-if" @@ -564,6 +567,21 @@ version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78cc372d058dcf6d5ecd98510e7fbc9e5aec4d21de70f65fea8fecebcd881bd4" +[[package]] +name = "git2" +version = "0.13.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f29229cc1b24c0e6062f6e742aa3e256492a5323365e5ed3413599f8a5eff7d6" +dependencies = [ + "bitflags", + "libc", + "libgit2-sys", + "log", + "openssl-probe", + "openssl-sys", + "url", +] + [[package]] name = "globset" version = "0.4.8" @@ -729,6 +747,15 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35" +[[package]] +name = "jobserver" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af25a77299a7f711a01975c35a6a424eb6862092cc2d6c72c4ed6cbc56dfc1fa" +dependencies = [ + "libc", +] + [[package]] name = "js-sys" version = "0.3.56" @@ -750,6 +777,46 @@ version = "0.2.113" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eef78b64d87775463c549fbd80e19249ef436ea3bf1de2a1eb7e717ec7fab1e9" +[[package]] +name = "libgit2-sys" +version = "0.12.26+1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19e1c899248e606fbfe68dcb31d8b0176ebab833b103824af31bddf4b7457494" +dependencies = [ + "cc", + "libc", + "libssh2-sys", + "libz-sys", + "openssl-sys", + "pkg-config", +] + +[[package]] +name = "libssh2-sys" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b094a36eb4b8b8c8a7b4b8ae43b2944502be3e59cd87687595cf6b0a71b3f4ca" +dependencies = [ + "cc", + "libc", + "libz-sys", + "openssl-sys", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "libz-sys" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de5435b8549c16d423ed0c03dbaafe57cf6c3344744f1242520d59c9d8ecec66" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linked-hash-map" version = "0.5.4" @@ -953,6 +1020,25 @@ version = "11.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.72" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e46109c383602735fa0a2e48dd2b7c892b048e1bf69e5c3b1d804b7d9c203cb" +dependencies = [ + "autocfg", + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "os_str_bytes" version = "6.0.0" @@ -1034,6 +1120,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkg-config" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58893f751c9b0412871a09abd62ecd2a00298c6c83befa223ef98c52aef40cbe" + [[package]] name = "plotters" version = "0.3.1" @@ -1911,6 +2003,12 @@ dependencies = [ "serde", ] +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.4" @@ -2123,10 +2221,10 @@ name = "xtask_codegen" version = "0.0.0" dependencies = [ "anyhow", + "git2", "pico-args", "proc-macro2", "quote", - "syn", "ungrammar", "ureq", "walkdir", diff --git a/xtask/codegen/Cargo.toml b/xtask/codegen/Cargo.toml index 93c1b268489..b09a4002d62 100644 --- a/xtask/codegen/Cargo.toml +++ b/xtask/codegen/Cargo.toml @@ -14,7 +14,4 @@ proc-macro2 = { version = "1.0.36", features = ["span-locations"] } ungrammar = "1.14.9" walkdir = "2.3.2" ureq = "2.4.0" - -[dependencies.syn] -version = "*" -features = ["parsing", "printing", "full"] +git2 = "0.13.25" diff --git a/xtask/codegen/src/formatter.rs b/xtask/codegen/src/formatter.rs index 59cbb89b2cd..e0f17fc17bb 100644 --- a/xtask/codegen/src/formatter.rs +++ b/xtask/codegen/src/formatter.rs @@ -1,175 +1,178 @@ -#![allow(deprecated)] - use std::{ - cell::RefCell, - cmp::Ordering, - collections::{BTreeMap, BTreeSet}, - fs::{create_dir_all, read_dir, read_to_string, remove_file, File}, - io::{self, Write}, + collections::{BTreeMap, BTreeSet, HashSet, VecDeque}, + env, + fs::{create_dir_all, read_dir, remove_file, File}, + io::Write, path::{Path, PathBuf}, - rc::Rc, }; +use git2::{Repository, Status, StatusOptions}; use proc_macro2::{Ident, Span}; use quote::quote; -use syn::{parse_file, spanned::Spanned}; use xtask::project_root; use crate::ast::load_ast; -#[deprecated] -struct FileStore { - use_items: Vec, - impl_items: BTreeMap, -} - -#[deprecated] -type FileSet = Rc>; - -#[deprecated] -#[derive(Default)] -struct NodeIndex { - entries: BTreeMap, - files: BTreeMap, -} - -#[deprecated] -struct NodeIndexEntry { - use_items: Vec, - impl_item: String, - file: FileSet, +struct GitRepo { + repo: Repository, + allow_staged: bool, + staged: HashSet, + dirty: HashSet, } -#[deprecated] -fn load_source(index: &mut NodeIndex, path: &Path) { - let code_str = read_to_string(path).unwrap(); - let code = parse_file(&code_str) - .unwrap_or_else(|err| panic!("failed to parse {}: {:?}", path.display(), err)); - - let mut use_items: Vec = Vec::new(); - let mut impl_items: BTreeMap = BTreeMap::new(); - - for item in code.items { - match item { - syn::Item::Use(use_item) => { - use_items.push(slice_source(&code_str, &use_item)); +impl GitRepo { + fn open() -> Self { + let root = project_root(); + let repo = Repository::discover(&root).expect("failed to open git repo"); + + let mut allow_staged = false; + let mut allow_dirty = false; + for arg in env::args() { + match arg.as_str() { + "--allow-staged" => { + allow_staged = true; + } + "--allow-dirty" => { + allow_dirty = true; + } + _ => {} } + } - syn::Item::Impl(impl_item) => { - if let Some((_, path, _)) = &impl_item.trait_ { - if let Some(segment) = path.segments.last() { - if segment.ident == "ToFormatElement" { - if let syn::Type::Path(path) = &*impl_item.self_ty { - if let Some(segment) = path.path.segments.last() { - impl_items.insert( - segment.ident.to_string(), - slice_source(&code_str, &impl_item), - ); - } - } + let mut repo_opts = StatusOptions::new(); + repo_opts.include_ignored(false); + + let statuses = repo + .statuses(Some(&mut repo_opts)) + .expect("failed to read repository status"); + + let mut staged = HashSet::new(); + let mut dirty = HashSet::new(); + + for status in statuses.iter() { + if let Some(path) = status.path() { + match status.status() { + Status::CURRENT => (), + Status::INDEX_NEW + | Status::INDEX_MODIFIED + | Status::INDEX_DELETED + | Status::INDEX_RENAMED + | Status::INDEX_TYPECHANGE => { + if !allow_staged { + staged.insert(root.join(path)); } } - } + _ => { + if !allow_dirty { + dirty.insert(root.join(path)); + } + } + }; } - - _ => {} } - } - let file = Rc::new(RefCell::new(FileStore { - use_items: use_items.clone(), - impl_items: impl_items.clone(), - })); + drop(statuses); - index.files.insert(path.into(), Rc::clone(&file)); - - for (key, impl_item) in impl_items { - index.entries.insert( - key, - NodeIndexEntry { - use_items: use_items.clone(), - impl_item, - file: Rc::clone(&file), - }, - ); + Self { + repo, + allow_staged, + staged, + dirty, + } } -} - -#[deprecated] -fn slice_source(source: &str, node: &impl Spanned) -> String { - let span = node.span(); - let start = span.start(); - let end = span.end(); - - let mut buffer = String::new(); - for (line_index, line_content) in source.lines().enumerate() { - let line_number = line_index + 1; - let (offset, line_content) = match line_number.cmp(&start.line) { - Ordering::Less => continue, - Ordering::Equal => (start.column, &line_content[start.column..]), - Ordering::Greater => (0, line_content), - }; - let line_content = match line_number.cmp(&end.line) { - Ordering::Less => line_content, - Ordering::Equal => &line_content[..end.column - offset], - Ordering::Greater => break, - }; - - buffer.push_str(line_content); - buffer.push('\n'); + fn check_path(&self, path: &Path) { + if self.dirty.contains(path) { + panic!("Codegen would overwrite '{}' but it has uncommited changes. Commit the file to git, or pass --allow-dirty to the command to proceed anyway", path.display()); + } + if self.staged.contains(path) { + panic!("Codegen would overwrite '{}' but it has uncommited changes. Commit the file to git, or pass --allow-staged to the command to proceed anyway", path.display()); + } } - buffer -} - -#[deprecated] -fn traverse_directory(index: &mut NodeIndex, path: &Path) -> io::Result<()> { - for entry in read_dir(path)? { - let entry = match entry { - Ok(entry) => entry, - Err(_) => continue, - }; - - let kind = match entry.file_type() { - Ok(kind) => kind, - Err(_) => continue, - }; - - if kind.is_file() { - load_source(index, &entry.path()); - continue; + fn stage_paths(&self, paths: &[PathBuf]) { + // Do not overwrite a version of the file + // that's potentially already staged + if self.allow_staged { + return; } - if kind.is_dir() { - traverse_directory(index, &entry.path()).ok(); - continue; - } + let root = project_root(); + self.repo + .index() + .expect("could not open index for git repository") + .update_all( + paths.iter().map(|path| { + path.strip_prefix(&root).unwrap_or_else(|err| { + panic!( + "path '{}' is not inside of project '{}': {}", + path.display(), + root.display(), + err, + ) + }) + }), + None, + ) + .expect("failed to stage updated files"); } - - Ok(()) } struct ModuleIndex { root: PathBuf, modules: BTreeMap>, + unused_files: HashSet, } impl ModuleIndex { fn new(root: PathBuf) -> Self { + let mut unused_files = HashSet::new(); + let mut queue = VecDeque::new(); + + queue.push_back(root.join("js")); + queue.push_back(root.join("ts")); + + while let Some(dir) = queue.pop_front() { + let iter = read_dir(&dir) + .unwrap_or_else(|err| panic!("failed to read '{}': {}", dir.display(), err)); + + for entry in iter { + let entry = entry.expect("failed to read DirEntry"); + + let path = entry.path(); + let file_type = entry.file_type().unwrap_or_else(|err| { + panic!("failed to read file type of '{}': {}", path.display(), err) + }); + + if file_type.is_dir() { + queue.push_back(path); + continue; + } + + if file_type.is_file() { + unused_files.insert(path); + } + } + } + Self { root, modules: BTreeMap::default(), + unused_files, } } /// Add a new module to the index - fn insert(&mut self, path: &Path) { + fn insert(&mut self, repo: &GitRepo, path: &Path) { + self.unused_files.remove(path); + // Walk up from the module file towards the root let mut parent = path.parent(); let mut file_stem = path.file_stem(); while let (Some(path), Some(stem)) = (parent, file_stem) { + repo.check_path(&path.join("mod.rs")); + // Insert each module into its parent let stem = stem.to_str().unwrap().to_owned(); self.modules.entry(path.into()).or_default().insert(stem); @@ -186,7 +189,7 @@ impl ModuleIndex { /// Create all the mod.rs files needed to import /// all the modules in the index up to the root - fn print(self) { + fn print(mut self, stage: &mut Vec) { for (path, imports) in self.modules { let mut content = String::new(); @@ -208,6 +211,16 @@ impl ModuleIndex { let path = path.join("mod.rs"); let mut file = File::create(&path).unwrap(); file.write_all(content.as_bytes()).unwrap(); + drop(file); + + self.unused_files.remove(&path); + stage.push(path); + } + + for file in self.unused_files { + remove_file(&file) + .unwrap_or_else(|err| panic!("failed to delete '{}': {}", file.display(), err)); + stage.push(file); } } } @@ -220,15 +233,7 @@ enum NodeKind { } pub fn generate_formatter() { - let mut index = NodeIndex::default(); - - if true { - traverse_directory( - &mut index, - &project_root().join("crates/rome_formatter/src/old"), - ) - .ok(); - } + let repo = GitRepo::open(); let ast = load_ast(); @@ -263,11 +268,13 @@ pub fn generate_formatter() { ) })); + let mut stage = Vec::new(); + // Create a default implementation for theses nodes only if // the file doesn't already exist for (kind, name) in names { let path = name_to_path(&kind, &name); - modules.insert(&path); + modules.insert(&repo, &path); // Union nodes except for AnyFunction and AnyClass have a generated // implementation, the codegen will always overwrite any existing file @@ -279,98 +286,79 @@ pub fn generate_formatter() { continue; } + repo.check_path(&path); + let dir = path.parent().unwrap(); create_dir_all(dir).unwrap(); - let tokens = match index.entries.remove(&name) { - Some(entry) if !allow_overwrite => { - let use_items = entry.use_items; - let impl_item = entry.impl_item; + let id = Ident::new(&name, Span::call_site()); - entry.file.borrow_mut().impl_items.remove(&name); + // Generate a default implementation of ToFormatElement using format_list on + // non-separated lists, to_format_element on the wrapped node for unions and + // format_verbatim for all the other nodes + let tokens = match kind { + NodeKind::List { separated: false } => quote! { + use crate::{FormatElement, FormatResult, Formatter, ToFormatElement}; + use rslint_parser::ast::#id; - format!("{}\n{}", use_items.join("\n"), impl_item) - } - _ => { - let id = Ident::new(&name, Span::call_site()); - - // Generate a default implementation of ToFormatElement using format_list on - // non-separated lists, to_format_element on the wrapped node for unions and - // format_verbatim for all the other nodes - let tokens = match kind { - NodeKind::List { separated: false } => quote! { - use crate::{FormatElement, FormatResult, Formatter, ToFormatElement}; - use rslint_parser::ast::#id; - - impl ToFormatElement for #id { - fn to_format_element(&self, formatter: &Formatter) -> FormatResult { - Ok(formatter.format_list(self.clone())) - } - } - }, - NodeKind::Node | NodeKind::Unknown | NodeKind::List { separated: true } => { - quote! { - use crate::{FormatElement, FormatResult, Formatter, ToFormatElement}; - use rslint_parser::{ast::#id, AstNode}; - - impl ToFormatElement for #id { - fn to_format_element(&self, formatter: &Formatter) -> FormatResult { - Ok(formatter.format_verbatim(self.syntax())) - } - } + impl ToFormatElement for #id { + fn to_format_element(&self, formatter: &Formatter) -> FormatResult { + Ok(formatter.format_list(self.clone())) + } + } + }, + NodeKind::Node | NodeKind::Unknown | NodeKind::List { separated: true } => { + quote! { + use crate::{FormatElement, FormatResult, Formatter, ToFormatElement}; + use rslint_parser::{ast::#id, AstNode}; + + impl ToFormatElement for #id { + fn to_format_element(&self, formatter: &Formatter) -> FormatResult { + Ok(formatter.format_verbatim(self.syntax())) } } - NodeKind::Union { variants } => { - // For each variant of the union call to_format_element on the wrapped node - let match_arms: Vec<_> = variants - .into_iter() - .map(|variant| { - let variant = Ident::new(&variant, Span::call_site()); - quote! { Self::#variant(node) => node.to_format_element(formatter), } - }) - .collect(); - - quote! { - use crate::{FormatElement, FormatResult, Formatter, ToFormatElement}; - use rslint_parser::ast::#id; - - impl ToFormatElement for #id { - fn to_format_element(&self, formatter: &Formatter) -> FormatResult { - match self { - #( #match_arms )* - } - } + } + } + NodeKind::Union { variants } => { + // For each variant of the union call to_format_element on the wrapped node + let match_arms: Vec<_> = variants + .into_iter() + .map(|variant| { + let variant = Ident::new(&variant, Span::call_site()); + quote! { Self::#variant(node) => node.to_format_element(formatter), } + }) + .collect(); + + quote! { + use crate::{FormatElement, FormatResult, Formatter, ToFormatElement}; + use rslint_parser::ast::#id; + + impl ToFormatElement for #id { + fn to_format_element(&self, formatter: &Formatter) -> FormatResult { + match self { + #( #match_arms )* } } } - }; - - if allow_overwrite { - xtask::reformat(tokens).unwrap() - } else { - xtask::reformat_without_preamble(tokens).unwrap() } } }; + let tokens = if allow_overwrite { + xtask::reformat(tokens).unwrap() + } else { + xtask::reformat_without_preamble(tokens).unwrap() + }; + let mut file = File::create(&path).unwrap(); file.write_all(tokens.as_bytes()).unwrap(); - } - - for (path, entry) in index.files { - let entry = entry.borrow(); - if entry.impl_items.is_empty() { - remove_file(path).unwrap(); - } else { - let impl_items: Vec<_> = entry.impl_items.values().cloned().collect(); - let tokens = format!("{}\n{}", entry.use_items.join("\n"), impl_items.join("\n")); + drop(file); - let mut file = File::create(&path).unwrap(); - file.write_all(tokens.as_bytes()).unwrap(); - } + stage.push(path); } - modules.print(); + modules.print(&mut stage); + repo.stage_paths(&stage); } enum NodeLanguage {