Update code to work with new embed syntax

Cargo.toml

@@ -33,6 +33,7 @@ serde_json = "1.0"
 rayon = "0.8.0"
 regex = "0.2.1"
 getopts = "0.2"
+pretty_assertions = "0.4.0"
 
 [features]
 

examples/syntest.rs

@@ -1,6 +1,12 @@
 //! An example of using syntect for testing syntax definitions.
 //! Basically exactly the same as what Sublime Text can do,
 //! but without needing ST installed
+// To run tests only for a particular package, while showing the operations, you could use:
+// cargo run --example syntest -- --debug testdata/Packages/Makefile/
+// to specify that the syntax definitions should be parsed instead of loaded from the dump file,
+// you can tell it where to parse them from - the following will execute only 1 syntax test after
+// parsing the sublime-syntax files in the JavaScript folder:
+// cargo run --example syntest testdata/Packages/JavaScript/syntax_test_json.json testdata/Packages/JavaScript/
 extern crate syntect;
 extern crate walkdir;
 #[macro_use]
@@ -123,7 +129,8 @@ fn process_assertions(assertion: &AssertionRange, test_against_line_scopes: &Vec
 }
 
 /// If `parse_test_lines` is `false` then lines that only contain assertions are not parsed
-fn test_file(ss: &SyntaxSet, path: &Path, parse_test_lines: bool) -> Result<SyntaxTestFileResult, SyntaxTestHeaderError> {
+fn test_file(ss: &SyntaxSet, path: &Path, parse_test_lines: bool, debug: bool) -> Result<SyntaxTestFileResult, SyntaxTestHeaderError> {
+    use syntect::util::debug_print_ops;
     let f = File::open(path).unwrap();
     let mut reader = BufReader::new(f);
     let mut line = String::new();
@@ -189,7 +196,17 @@ fn test_file(ss: &SyntaxSet, path: &Path, parse_test_lines: bool) -> Result<Synt
                 test_against_line_number = current_line_number;
                 previous_non_assertion_line = line.to_string();
             }
+            if debug && !line_only_has_assertion {
+                println!("-- debugging line {} -- scope stack: {:?}", current_line_number, stack);
+            }
             let ops = state.parse_line(&line);
+            if debug && !line_only_has_assertion {
+                if ops.is_empty() && !line.is_empty() {
+                    println!("no operations for this line...");
+                } else {
+                    debug_print_ops(&line, &ops);
+                }
+            }
             let mut col: usize = 0;
             for (s, op) in ScopeRegionIterator::new(&ops, &line) {
                 stack.apply(op);
@@ -227,7 +244,12 @@ fn test_file(ss: &SyntaxSet, path: &Path, parse_test_lines: bool) -> Result<Synt
 }
 
 fn main() {
-    let args: Vec<String> = std::env::args().collect();
+    let mut args: Vec<String> = std::env::args().collect();
+    let debug_arg = args.iter().position(|s| s == "--debug");
+    if debug_arg.is_some() {
+        args.remove(debug_arg.unwrap());
+    }
+
     let tests_path = if args.len() < 2 {
         "."
     } else {
@@ -254,21 +276,21 @@ fn main() {
         ss.link_syntaxes();
     }
 
-    let exit_code = recursive_walk(&ss, &tests_path);
+    let exit_code = recursive_walk(&ss, &tests_path, debug_arg.is_some());
     println!("exiting with code {}", exit_code);
     std::process::exit(exit_code);
 
 }
 
 
-fn recursive_walk(ss: &SyntaxSet, path: &str) -> i32 {
+fn recursive_walk(ss: &SyntaxSet, path: &str, debug: bool) -> i32 {
     let mut exit_code: i32 = 0; // exit with code 0 by default, if all tests pass
     let walker = WalkDir::new(path).into_iter();
     for entry in walker.filter_entry(|e|e.file_type().is_dir() || is_a_syntax_test_file(e)) {
         let entry = entry.unwrap();
         if entry.file_type().is_file() {
             println!("Testing file {}", entry.path().display());
-            let result = test_file(&ss, entry.path(), true);
+            let result = test_file(&ss, entry.path(), true, debug);
             println!("{:?}", result);
             if exit_code != 2 { // leave exit code 2 if there was an error
                 if let Err(_) = result { // set exit code 2 if there was an error

src/lib.rs

@@ -41,6 +41,9 @@ extern crate serde;
 #[macro_use]
 extern crate serde_derive;
 extern crate serde_json;
+#[cfg(test)]
+#[macro_use]
+extern crate pretty_assertions;
 pub mod highlighting;
 pub mod parsing;
 pub mod util;

src/parsing/parser.rs

@@ -159,9 +159,9 @@ impl ParseState {
             let context_chain = {
                 let proto_start = self.proto_starts.last().cloned().unwrap_or(0);
                 // Sublime applies with_prototypes from bottom to top
-                let with_prototypes = self.stack[proto_start..].iter().filter_map(|lvl| lvl.prototype.as_ref().cloned()).map(|ctx| (true, ctx));
-                let cur_prototype = prototype.into_iter().map(|ctx| (false, ctx));
-                let cur_context = Some((false, cur_level.context.clone())).into_iter();
+                let with_prototypes = self.stack[proto_start..].iter().filter_map(|lvl| lvl.prototype.as_ref().map(|ctx| (true, ctx.clone(), lvl.captures.as_ref())));
+                let cur_prototype = prototype.into_iter().map(|ctx| (false, ctx, None));
+                let cur_context = Some((false, cur_level.context.clone(), cur_level.captures.as_ref())).into_iter();
                 with_prototypes.chain(cur_prototype).chain(cur_context)
             };
 
@@ -172,7 +172,7 @@ impl ParseState {
             let mut match_from_with_proto = false;
             let mut cur_match: Option<RegexMatch> = None;
 
-            for (from_with_proto, ctx) in context_chain {
+            for (from_with_proto, ctx, captures) in context_chain {
                 for (pat_context_ptr, pat_index) in context_iter(ctx) {
                     let mut pat_context = pat_context_ptr.borrow_mut();
                     let match_pat = pat_context.match_at_mut(pat_index);
@@ -208,8 +208,8 @@ impl ParseState {
                     }
 
                     match_pat.ensure_compiled_if_possible();
-                    let refs_regex = if match_pat.has_captures && cur_level.captures.is_some() {
-                        let &(ref region, ref s) = cur_level.captures.as_ref().unwrap();
+                    let refs_regex = if match_pat.has_captures && captures.is_some() {
+                        let &(ref region, ref s) = captures.unwrap();
                         Some(match_pat.compile_with_refs(region, s))
                     } else {
                         None
@@ -451,15 +451,23 @@ impl ParseState {
             MatchOperation::None => return false,
         };
         for (i, r) in ctx_refs.iter().enumerate() {
-            let proto = if i == 0 {
+            // if a with_prototype was specified, and multiple contexts were pushed,
+            // then the with_prototype applies only to the last context pushed, i.e.
+            // top most on the stack after all the contexts are pushed - this is also
+            // referred to as the "target" of the push by sublimehq - see
+            // https://forum.sublimetext.com/t/dev-build-3111/19240/17 for more info
+            let proto = if i == ctx_refs.len() - 1 {
                 pat.with_prototype.clone()
             } else {
                 None
             };
             let ctx_ptr = r.resolve();
             let captures = {
-                let ctx = ctx_ptr.borrow();
-                if ctx.uses_backrefs {
+                let mut uses_backrefs = ctx_ptr.borrow().uses_backrefs;
+                if let Some(ref proto) = proto {
+                    uses_backrefs = uses_backrefs || proto.borrow().uses_backrefs;
+                }
+                if uses_backrefs {
                     Some((regions.clone(), line.to_owned()))
                 } else {
                     None
@@ -535,6 +543,7 @@ mod tests {
         test_stack.push(Scope::new("text.html.ruby").unwrap());
         test_stack.push(Scope::new("text.html.basic").unwrap());
         test_stack.push(Scope::new("source.js.embedded.html").unwrap());
+        test_stack.push(Scope::new("source.js").unwrap());
         test_stack.push(Scope::new("string.quoted.single.js").unwrap());
         test_stack.push(Scope::new("source.ruby.rails.embedded.html").unwrap());
         test_stack.push(Scope::new("meta.function.parameters.ruby").unwrap());
@@ -719,4 +728,28 @@ contexts:
         debug_print_ops(line, &ops);
         ops
     }
+
+    #[test]
+    fn can_parse_issue120() {
+        let ps = SyntaxSet::load_from_folder("testdata").unwrap();
+        let syntax = ps.find_syntax_by_name("Embed_Escape Used by tests in src/parsing/parser.rs").unwrap();
+
+        let line1 = "\"abctest\" foobar";
+        let expect1 = [
+            "<meta.attribute-with-value.style.html>, <string.quoted.double>, <punctuation.definition.string.begin.html>",
+            "<meta.attribute-with-value.style.html>, <source.css>",
+            "<meta.attribute-with-value.style.html>, <string.quoted.double>, <punctuation.definition.string.end.html>",
+            "<meta.attribute-with-value.style.html>, <source.css>, <test.embedded>",
+            "<top-level.test>",
+        ];
+        expect_scope_stacks_with_syntax(&line1, &expect1, syntax.to_owned());
+
+        let line2 = ">abctest</style>foobar";
+        let expect2 = [
+            "<meta.tag.style.begin.html>, <punctuation.definition.tag.end.html>",
+            "<source.css.embedded.html>, <test.embedded>",
+            "<top-level.test>",
+        ];
+        expect_scope_stacks_with_syntax(&line2, &expect2, syntax.to_owned());
+    }
 }

src/parsing/syntax_definition.rs

@@ -57,6 +57,20 @@ pub struct Context {
     pub patterns: Vec<Pattern>,
 }
 
+impl Context {
+    pub fn new(meta_include_prototype: bool) -> Context {
+        Context {
+            meta_scope: Vec::new(),
+            meta_content_scope: Vec::new(),
+            meta_include_prototype: meta_include_prototype,
+            clear_scopes: None,
+            uses_backrefs: false,
+            patterns: Vec::new(),
+            prototype: None,
+        }
+    }
+}
+
 #[derive(Debug, Eq, PartialEq, Serialize, Deserialize)]
 pub enum Pattern {
     Match(MatchPattern),

src/parsing/syntax_set.rs

@@ -95,7 +95,6 @@ impl SyntaxSet {
         for entry in WalkDir::new(folder).sort_by(|a, b| a.cmp(b)) {
             let entry = entry.map_err(LoadingError::WalkDir)?;
             if entry.path().extension().map_or(false, |e| e == "sublime-syntax") {
-                // println!("{}", entry.path().display());
                 let syntax = load_syntax_file(entry.path(), lines_include_newline)?;
                 if let Some(path_str) = entry.path().to_str() {
                     self.path_syntaxes.push((path_str.to_string(), self.syntaxes.len()));