Update code to work with new embed syntax

Cargo.toml

@@ -33,6 +33,7 @@ serde_json = "1.0"
 rayon = "0.8.0"
 regex = "0.2.1"
 getopts = "0.2"
+pretty_assertions = "0.4.0"
 
 [features]
 

src/lib.rs

@@ -41,6 +41,9 @@ extern crate serde;
 #[macro_use]
 extern crate serde_derive;
 extern crate serde_json;
+#[cfg(test)]
+#[macro_use]
+extern crate pretty_assertions;
 pub mod highlighting;
 pub mod parsing;
 pub mod util;

src/parsing/parser.rs

@@ -451,8 +451,26 @@ impl ParseState {
             MatchOperation::None => return false,
         };
         for (i, r) in ctx_refs.iter().enumerate() {
-            let proto = if i == 0 {
-                pat.with_prototype.clone()
+            let proto = if i == ctx_refs.len() - 1 { // https://forum.sublimetext.com/t/dev-build-3111/19240/17
+                if pat.with_prototype.is_some() {
+                    let p = pat.with_prototype.clone().unwrap();
+                    {
+                        let mut b = p.borrow_mut();
+                        for pattern in b.patterns.iter_mut() {
+                            match pattern {
+                                &mut Pattern::Match(ref mut match_pat) => {
+                                    if match_pat.has_captures {
+                                        match_pat.regex = Some(match_pat.compile_with_refs(regions, line));
+                                    }
+                                },
+                                &mut Pattern::Include(_) => {}, // TODO: included contexts can also contain backrefs in their match patterns - see https://github.com/trishume/syntect/issues/104
+                            }
+                        }
+                    }
+                    Some(p)
+                } else {
+                    None
+                }
             } else {
                 None
             };
@@ -719,4 +737,28 @@ contexts:
         debug_print_ops(line, &ops);
         ops
     }
+
+    #[test]
+    fn can_parse_issue120() {
+        let ps = SyntaxSet::load_from_folder("testdata").unwrap();
+        let syntax = ps.find_syntax_by_name("Embed_Escape Used by tests in src/parsing/parser.rs").unwrap();
+
+        let line1 = "\"abctest\" foobar";
+        let expect1 = [
+            "<meta.attribute-with-value.style.html>, <string.quoted.double>, <punctuation.definition.string.begin.html>",
+            "<meta.attribute-with-value.style.html>, <source.css>",
+            "<meta.attribute-with-value.style.html>, <string.quoted.double>, <punctuation.definition.string.end.html>",
+            "<meta.attribute-with-value.style.html>, <source.css>, <test.embedded>",
+            "<top-level.test>",
+        ];
+        expect_scope_stacks_with_syntax(&line1, &expect1, syntax.to_owned());
+
+        let line2 = ">abctest</style>foobar";
+        let expect2 = [
+            "<meta.tag.style.begin.html>, <punctuation.definition.tag.end.html>",
+            "<source.css.embedded.html>, <test.embedded>",
+            "<top-level.test>",
+        ];
+        expect_scope_stacks_with_syntax(&line2, &expect2, syntax.to_owned());
+    }
 }

src/parsing/syntax_definition.rs

@@ -57,6 +57,20 @@ pub struct Context {
     pub patterns: Vec<Pattern>,
 }
 
+impl Context {
+    pub fn new(meta_include_prototype: bool) -> Context {
+        Context {
+            meta_scope: Vec::new(),
+            meta_content_scope: Vec::new(),
+            meta_include_prototype: meta_include_prototype,
+            clear_scopes: None,
+            uses_backrefs: false,
+            patterns: Vec::new(),
+            prototype: None,
+        }
+    }
+}
+
 #[derive(Debug, Eq, PartialEq, Serialize, Deserialize)]
 pub enum Pattern {
     Match(MatchPattern),

src/parsing/syntax_set.rs

@@ -95,7 +95,6 @@ impl SyntaxSet {
         for entry in WalkDir::new(folder).sort_by(|a, b| a.cmp(b)) {
             let entry = entry.map_err(LoadingError::WalkDir)?;
             if entry.path().extension().map_or(false, |e| e == "sublime-syntax") {
-                // println!("{}", entry.path().display());
                 let syntax = load_syntax_file(entry.path(), lines_include_newline)?;
                 if let Some(path_str) = entry.path().to_str() {
                     self.path_syntaxes.push((path_str.to_string(), self.syntaxes.len()));

src/parsing/yaml_load.rs

@@ -13,7 +13,7 @@ use std::ops::DerefMut;
 pub enum ParseSyntaxError {
     /// Invalid YAML file syntax, or at least something yaml_rust can't handle
     InvalidYaml(ScanError),
-    /// The file must contain at least on YAML document
+    /// The file must contain at least one YAML document
     EmptyFile,
     /// Some keys are required for something to be a valid `.sublime-syntax`
     MissingMandatoryKey(&'static str),
@@ -152,15 +152,8 @@ impl SyntaxDefinition {
                      state: &mut ParserState,
                      is_prototype: bool)
                      -> Result<ContextPtr, ParseSyntaxError> {
-        let mut context = Context {
-            meta_scope: Vec::new(),
-            meta_content_scope: Vec::new(),
-            meta_include_prototype: !is_prototype,
-            clear_scopes: None,
-            uses_backrefs: false,
-            patterns: Vec::new(),
-            prototype: None,
-        };
+        let mut context = Context::new(!is_prototype);
+
         for y in vec.iter() {
             let map = y.as_hash().ok_or(ParseSyntaxError::TypeMismatch)?;
 
@@ -265,12 +258,13 @@ impl SyntaxDefinition {
             .map(|s| str_to_scopes(s, state.scope_repo))
             .unwrap_or_else(|| Ok(vec![]))?;
 
+
         let captures = if let Ok(map) = get_key(map, "captures", |x| x.as_hash()) {
             let mut res_map = Vec::new();
             for (key, value) in map.iter() {
                 if let (Some(key_int), Some(val_str)) = (key.as_i64(), value.as_str()) {
                     res_map.push((key_int as usize,
-                                   str_to_scopes(val_str, state.scope_repo)?));
+                                  str_to_scopes(val_str, state.scope_repo)?));
                 }
             }
             Some(res_map)
@@ -287,13 +281,59 @@ impl SyntaxDefinition {
             MatchOperation::Push(SyntaxDefinition::parse_pushargs(y, state)?)
         } else if let Ok(y) = get_key(map, "set", Some) {
             MatchOperation::Set(SyntaxDefinition::parse_pushargs(y, state)?)
+        } else if let Ok(y) = get_key(map, "embed", Some) {
+            let mut embed_context = vec!();
+            // Same as push so we translate it to what it would be
+            let mut commands = Hash::new();
+            if let Ok(s) = get_key(map, "embed_scope", Some) {
+                commands.insert(Yaml::String("meta_content_scope".to_string()), s.clone());
+                embed_context.push(Yaml::Hash(commands));
+                commands = Hash::new();
+            }
+            commands.insert(Yaml::String("include".to_string()), y.clone());
+            embed_context.push(Yaml::Hash(commands));
+            let embedded_context = SyntaxDefinition::parse_context(
+                &embed_context,
+                state,
+                false
+            )?;
+
+            if let Ok(v) = get_key(map, "escape", Some) {
+                let mut match_map = Hash::new();
+                match_map.insert(Yaml::String("match".to_string()), v.clone());
+                match_map.insert(Yaml::String("pop".to_string()), Yaml::Boolean(true));
+                if let Ok(y) = get_key(map, "escape_captures", Some) {
+                    match_map.insert(Yaml::String("captures".to_string()), y.clone());
+                }
+                let escape_context = SyntaxDefinition::parse_context(
+                    &[Yaml::Hash(match_map)],
+                    state,
+                    false
+                )?;
+                MatchOperation::Push(vec![ContextReference::Inline(escape_context), ContextReference::Inline(embedded_context)])
+            } else {
+                return Err(ParseSyntaxError::MissingMandatoryKey("escape"));
+            }
+
         } else {
             MatchOperation::None
         };
 
         let with_prototype = if let Ok(v) = get_key(map, "with_prototype", |x| x.as_vec()) {
             // should a with_prototype include the prototype? I don't think so.
-            Some(SyntaxDefinition::parse_context(v, state, true)?)
+            Some(Self::parse_context(v, state, true)?)
+        } else if let Ok(v) = get_key(map, "escape", Some) {
+            let mut context = Context::new(false);
+            let mut match_map = Hash::new();
+            match_map.insert(Yaml::String("match".to_string()), Yaml::String(format!("(?={})", v.as_str().unwrap())));
+            match_map.insert(Yaml::String("pop".to_string()), Yaml::Boolean(true));
+            let pattern = SyntaxDefinition::parse_match_pattern(&match_map, state)?;
+            if pattern.has_captures {
+                context.uses_backrefs = true;
+            }
+            context.patterns.push(Pattern::Match(pattern));
+
+            Some(Rc::new(RefCell::new(context)))
         } else {
             None
         };
@@ -307,14 +347,15 @@ impl SyntaxDefinition {
             operation: operation,
             with_prototype: with_prototype,
         };
+
         Ok(pattern)
     }
 
     fn parse_pushargs(y: &Yaml,
                       state: &mut ParserState)
                       -> Result<Vec<ContextReference>, ParseSyntaxError> {
         // check for a push of multiple items
-        if y.as_vec().map_or(false, |v| !v.is_empty() && v[0].as_str().is_some()) {
+        if y.as_vec().map_or(false, |v| !v.is_empty() && (v[0].as_str().is_some() || (v[0].as_vec().is_some() && v[0].as_vec().unwrap()[0].as_hash().is_some()))) {
             // this works because Result implements FromIterator to handle the errors
             y.as_vec()
                 .unwrap()
@@ -621,6 +662,117 @@ mod tests {
         }
     }
 
+    #[test]
+    fn can_parse_embed_as_with_prototypes() {
+        let old_def = SyntaxDefinition::load_from_str(r#"
+        name: C
+        scope: source.c
+        file_extensions: [c, h]
+        variables:
+          ident: '[QY]+'
+        contexts:
+          main:
+            - match: '(>)\s*'
+              captures:
+                1: meta.tag.style.begin.html punctuation.definition.tag.end.html
+              push:
+                - [{ match: '(?i)(?=</style)', pop: true }]
+                - [{ meta_content_scope: 'source.css.embedded.html'}, { include: 'scope:source.css' }]
+              with_prototype:
+                - match: (?=(?i)(?=</style))
+                  pop: true
+        "#,false).unwrap();
+
+        let def_with_embed = SyntaxDefinition::load_from_str(r#"
+        name: C
+        scope: source.c
+        file_extensions: [c, h]
+        variables:
+          ident: '[QY]+'
+        contexts:
+          main:
+            - match: '(>)\s*'
+              captures:
+                1: meta.tag.style.begin.html punctuation.definition.tag.end.html
+              embed: scope:source.css
+              embed_scope: source.css.embedded.html
+              escape: (?i)(?=</style)
+        "#,false).unwrap();
+
+        assert_eq!(old_def.contexts["main"], def_with_embed.contexts["main"]);
+    }
+
+    #[test]
+    fn errors_on_embed_without_escape() {
+        let def = SyntaxDefinition::load_from_str(r#"
+        name: C
+        scope: source.c
+        file_extensions: [c, h]
+        variables:
+          ident: '[QY]+'
+        contexts:
+          main:
+            - match: '(>)\s*'
+              captures:
+                1: meta.tag.style.begin.html punctuation.definition.tag.end.html
+              embed: scope:source.css
+              embed_scope: source.css.embedded.html
+        "#,false);
+        assert!(def.is_err());
+        match def.unwrap_err() {
+            ParseSyntaxError::MissingMandatoryKey(key) => assert_eq!(key, "escape"),
+            _ => assert!(false, "Got unexpected ParseSyntaxError"),
+        }
+    }
+
+    #[test]
+    fn can_parse_ugly_yaml() {
+        let defn: SyntaxDefinition =
+            SyntaxDefinition::load_from_str("
+        name: LaTeX
+        scope: text.tex.latex
+        contexts:
+          main:
+            - match: '((\\\\)(?:framebox|makebox))\\b'
+              captures:
+                1: support.function.box.latex
+                2: punctuation.definition.backslash.latex
+              push:
+                - [{meta_scope: meta.function.box.latex}, {match: '', pop: true}]
+                - argument
+                - optional-arguments
+          argument:
+            - match: '\\{'
+              scope: punctuation.definition.group.brace.begin.latex
+            - match: '(?=\\S)'
+              pop: true
+          optional-arguments:
+            - match: '(?=\\S)'
+              pop: true
+        ",
+                                            false)
+                .unwrap();
+        assert_eq!(defn.name, "LaTeX");
+        let top_level_scope = Scope::new("text.tex.latex").unwrap();
+        assert_eq!(defn.scope, top_level_scope);
+
+        let first_pattern: &Pattern = &defn.contexts["main"].borrow().patterns[0];
+        match first_pattern {
+            &Pattern::Match(ref match_pat) => {
+                let m: &CaptureMapping = match_pat.captures.as_ref().expect("test failed");
+                assert_eq!(&m[0], &(1,vec![Scope::new("support.function.box.latex").unwrap()]));
+
+                //use parsing::syntax_definition::ContextReference::*;
+                // TODO: check the first pushed reference is Inline(...) and has a meta_scope of meta.function.box.latex
+                // TODO: check the second pushed reference is Named("argument".to_owned())
+                // TODO: check the third pushed reference is Named("optional-arguments".to_owned())
+
+                assert!(match_pat.with_prototype.is_none());
+            }
+            _ => assert!(false),
+        }
+    }
+
     #[test]
     fn can_rewrite_regex() {
         fn rewrite(s: &str) -> String {

testdata/embed_escape_test.sublime-syntax

@@ -0,0 +1,33 @@
+%YAML 1.2
+---
+name: Embed_Escape Used by tests in src/parsing/parser.rs
+scope: source.embed-test
+contexts:
+  main:
+    - match: (")
+      scope: meta.attribute-with-value.style.html string.quoted.double punctuation.definition.string.begin.html
+      embed: embedded_context
+      embed_scope: meta.attribute-with-value.style.html source.css
+      escape: '\1'
+      escape_captures:
+        0: meta.attribute-with-value.style.html string.quoted.double punctuation.definition.string.end.html
+    - match: '(>)\s*'
+      captures:
+        1: meta.tag.style.begin.html punctuation.definition.tag.end.html
+      embed: embedded_context
+      embed_scope: source.css.embedded.html
+      escape: (?i)(?=</style)
+    - match: '</style>'
+    - match: foobar
+      scope: top-level.test
+
+  embedded_context:
+    - match: a
+      scope: a
+      push: # prove that multiple context levels can be "escape"d
+        - match: b
+          push:
+            - match: c
+              push:
+                - match: 'test'
+                  scope: test.embedded

testdata/minimized_tests/syntax_test_aspmini.asp

@@ -2,7 +2,7 @@
 <%
  Class TestClass2 Public Sub TestSub () Response.Write("wow") End Sub End Class
 '^^^^^ meta.class.asp meta.class.identifier.asp storage.type.asp
-'      ^ meta.class.asp meta.class.body.asp meta.class.asp meta.class.identifier.asp
+'      ^ meta.class.asp meta.class.identifier.asp
 '                ^ meta.class.asp meta.class.body.asp
 %>
  <p>foobar</p>