Dont check in the grammar, but generate it at release time (#2145)

* Try including the generated grammar in-tree

* Use the lalrpop-generated hash to control rebuilding

* Maybe toybox is available on darwin

* Try generating on release

* Add tomlq to the dev shell

* Remove the precommit hook

* Simplify the build script

* Print a status message and use mktemp

* Review comments

core/build.rs

@@ -1,8 +1,36 @@
+use std::path::{Path, PathBuf};
+
 fn main() {
-    lalrpop::Configuration::new()
-        .use_cargo_dir_conventions()
-        .process_file("src/parser/grammar.lalrpop")
-        .unwrap();
+    let checked_in_grammar_path = Path::new(&concat!(
+        env!("CARGO_MANIFEST_DIR"),
+        "/src/parser/grammar.rs"
+    ));
+
+    let out_dir = PathBuf::from(std::env::var_os("OUT_DIR").expect("missing OUT_DIR variable"));
+    let out_parser_dir = out_dir.join("parser");
+    std::fs::create_dir_all(&out_parser_dir).expect("failed to create $OUT_DIR/parser");
+    // Running lalrpop can be expensive. When building from git, we generate the parser
+    // in this build script, but when publishing the crate we add the generated
+    // parser to the published crate at `src/parser/grammar.rs`.
+    //
+    // In order to have this build script work for both the published crate and the git
+    // version, we try to copy `src/parser/grammar.rs` into the same location in $OUT_DIR
+    // that lalrpop would generate the grammar. If that copy fails because `src/parser/grammar.rs`
+    // doesn't exist, we're probably building from git and so we generate the grammar.
+    match std::fs::copy(checked_in_grammar_path, out_parser_dir.join("grammar.rs")) {
+        Ok(_) => {
+            eprintln!("Found a pre-generated LALRPOP grammar, copying it over");
+        }
+        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
+            eprintln!("Generating a fresh LALRPOP grammar");
+            lalrpop::Configuration::new()
+                .use_cargo_dir_conventions()
+                .process_file("src/parser/grammar.lalrpop")
+                .unwrap();
+        }
+        Err(e) => panic!("{e}"),
+    }
+    println!("cargo:rerun-if-changed=src/parser/grammar.lalrpop");
 
     #[cfg(feature = "nix-experimental")]
     {

flake.nix

@@ -509,6 +509,7 @@
           pkgs.nodejs
           pkgs.yarn
           pkgs.yarn2nix
+          pkgs.yq
           pkgs.nodePackages.markdownlint-cli
           pkgs.python3
         ];

scripts/release.sh

@@ -421,6 +421,20 @@ for crate in "${crates_to_publish[@]}"; do
     cleanup_actions+=('git reset -- '"$crate/Cargo.toml")
 done
 
+# Generate the nickel grammar so that users of the published crate don't
+# have to deal with lalrpop being slow.
+# 
+# cargo check is the cheapest supported way to run just the build script
+# https://github.com/rust-lang/cargo/issues/7178
+# The output path is hard to predict (it contains a hash), so we find it
+# using a clean output directory and some globbing
+report_progress "Pre-generating the Nickel parser from the LALRPOP grammar"
+temp_target_dir=$(mktemp -d)
+cargo check -p nickel-lang-core --target-dir=$temp_target_dir
+cp $temp_target_dir/debug/build/nickel-lang-core-*/out/parser/grammar.rs core/src/parser/grammar.rs
+rm -rf $temp_target_dir || true
+git add core/src/parser/grammar.rs
+
 # Cargo requires to commit changes, but the last changes are temporary
 # work-arounds for the crates.io release that aren't supposed to stay. we'll
 # reset them later.