Add support for loading metadata files

.travis.yml

@@ -25,7 +25,7 @@ cache:
 
 script:
   - cargo build
-  - cargo test
+  - cargo test --features metadata
   - make assets
   - make syntest
   - rm -Rf examples

Cargo.toml

@@ -55,6 +55,8 @@ dump-create = ["flate2/default", "bincode"]
 dump-create-rs = ["flate2/rust_backend", "bincode"]
 
 parsing = ["onig", "regex-syntax", "fnv"]
+# Support for .tmPreferenes metadata files (indentation, comment syntax, etc)
+metadata = ["parsing"]
 # The `assets` feature enables inclusion of the default theme and syntax packages.
 # For `assets` to do anything, it requires one of `dump-load-rs` or `dump-load` to be set.
 assets = []

Makefile

@@ -16,7 +16,7 @@ $(SUBMODULES):
 assets: packs themes
 
 packs: $(SUBMODULES)
-	cargo run --example gendata -- synpack testdata/Packages assets/default_newlines.packdump assets/default_nonewlines.packdump
+	cargo run --features=metadata --example gendata -- synpack testdata/Packages assets/default_newlines.packdump assets/default_nonewlines.packdump assets/default_metadata.packdump testdata/DefaultPackage
 
 themes: $(SUBMODULES)
 	cargo run --example gendata -- themepack testdata assets/default.themedump

examples/gendata.rs

@@ -2,26 +2,34 @@
 //! syntect, not as a helpful example for beginners.
 //! Although it is a valid example for serializing syntaxes, you probably won't need
 //! to do this yourself unless you want to cache your own compiled grammars.
+//!
+//! An example of how this script is used to generate the pack files included
+//! with syntect can be found under `make packs` in the Makefile.
 extern crate syntect;
 use syntect::parsing::SyntaxSetBuilder;
 use syntect::highlighting::ThemeSet;
 use syntect::dumps::*;
 use std::env;
 
 fn usage_and_exit() -> ! {
-    println!("USAGE: gendata synpack source-dir newlines.packdump nonewlines.packdump\n
+    println!("USAGE: gendata synpack source-dir \
+              newlines.packdump nonewlines.packdump \
+              [metadata.packdump] [metadata extra-source-dir]\n       \
               gendata themepack source-dir themepack.themedump");
     ::std::process::exit(2);
 }
 
 fn main() {
 
     let mut a = env::args().skip(1);
-    match (a.next(), a.next(), a.next(), a.next()) {
+    match (a.next(), a.next(), a.next(), a.next(), a.next(), a.next()) {
         (Some(ref cmd),
          Some(ref package_dir),
          Some(ref packpath_newlines),
-         Some(ref packpath_nonewlines)) if cmd == "synpack" => {
+         Some(ref packpath_nonewlines),
+         ref _option_metapath,
+         ref _option_metasource,
+         ) if cmd == "synpack" => {
             let mut builder = SyntaxSetBuilder::new();
             builder.add_plain_text_syntax();
             builder.add_from_folder(package_dir, true).unwrap();
@@ -31,10 +39,26 @@ fn main() {
             let mut builder_nonewlines = SyntaxSetBuilder::new();
             builder_nonewlines.add_plain_text_syntax();
             builder_nonewlines.add_from_folder(package_dir, false).unwrap();
+
+            #[cfg(feature = "metadata")]
+            {
+                if let Some(metasource) = _option_metasource {
+                    builder_nonewlines.add_from_folder(metasource, false).unwrap();
+                }
+            }
+
             let ss_nonewlines = builder_nonewlines.build();
             dump_to_file(&ss_nonewlines, packpath_nonewlines).unwrap();
+
+            #[cfg(feature = "metadata")]
+            {
+                if let Some(metapath) = _option_metapath {
+                    dump_to_file(&ss_nonewlines.metadata(), metapath).unwrap();
+                }
+            }
+
         }
-        (Some(ref s), Some(ref theme_dir), Some(ref packpath), None) if s == "themepack" => {
+        (Some(ref s), Some(ref theme_dir), Some(ref packpath), ..) if s == "themepack" => {
             let ts = ThemeSet::load_from_folder(theme_dir).unwrap();
             dump_to_file(&ts, packpath).unwrap();
         }

src/dumps.rs

@@ -94,17 +94,36 @@ impl SyntaxSet {
     /// you can even use the fact that SyntaxDefinitions are serializable with
     /// the bincode crate to cache dumps of additional syntaxes yourself.
     pub fn load_defaults_nonewlines() -> SyntaxSet {
-        let ss: SyntaxSet = from_binary(include_bytes!("../assets/default_nonewlines.\
-                                                             packdump"));
-        ss
+
+        #[cfg(feature = "metadata")]
+        {
+            let mut ps: SyntaxSet = from_binary(include_bytes!("../assets/default_nonewlines.packdump"));
+            let metadata = from_binary(include_bytes!("../assets/default_metadata.packdump"));
+            ps.metadata = metadata;
+            ps
+        }
+        #[cfg(not(feature = "metadata"))]
+        {
+            from_binary(include_bytes!("../assets/default_nonewlines.packdump"))
+        }
     }
 
     /// Same as `load_defaults_nonewlines` but for parsing line strings with newlines at the end.
     /// These are separate methods because thanks to linker garbage collection, only the serialized
     /// dumps for the method(s) you call will be included in the binary (each is ~200kb for now).
     pub fn load_defaults_newlines() -> SyntaxSet {
-        let ss: SyntaxSet = from_binary(include_bytes!("../assets/default_newlines.packdump"));
-        ss
+
+        #[cfg(feature = "metadata")]
+        {
+            let mut ps: SyntaxSet = from_binary(include_bytes!("../assets/default_newlines.packdump"));
+            let metadata = from_binary(include_bytes!("../assets/default_metadata.packdump"));
+            ps.metadata = metadata;
+            ps
+        }
+        #[cfg(not(feature = "metadata"))]
+        {
+            from_binary(include_bytes!("../assets/default_newlines.packdump"))
+        }
     }
 }
 

src/highlighting/mod.rs

@@ -3,7 +3,7 @@
 //! settings like selection color, `ThemeSet` for loading themes,
 //! as well as things starting with `Highlight` for how to highlight text.
 mod selector;
-mod settings;
+pub(crate) mod settings;
 mod style;
 mod theme;
 mod highlighter;

src/lib.rs

@@ -59,6 +59,9 @@ mod escape;
 use std::io::Error as IoError;
 use std::error::Error;
 use std::fmt;
+
+#[cfg(feature = "metadata")]
+use serde_json::Error as JsonError;
 #[cfg(all(feature = "yaml-load", feature = "parsing"))]
 use parsing::ParseSyntaxError;
 use highlighting::{ParseThemeError, SettingsError};
@@ -73,6 +76,9 @@ pub enum LoadingError {
     /// a syntax file was invalid in some way
     #[cfg(feature = "yaml-load")]
     ParseSyntax(ParseSyntaxError, Option<String>),
+    /// a metadata file was invalid in some way
+    #[cfg(feature = "metadata")]
+    ParseMetadata(JsonError),
     /// a theme file was invalid in some way
     ParseTheme(ParseThemeError),
     /// a theme's Plist syntax was invalid in some way
@@ -100,6 +106,13 @@ impl From<ParseThemeError> for LoadingError {
     }
 }
 
+#[cfg(feature = "metadata")]
+impl From<JsonError> for LoadingError {
+    fn from(src: JsonError) -> LoadingError {
+        LoadingError::ParseMetadata(src)
+    }
+}
+
 #[cfg(all(feature = "yaml-load", feature = "parsing"))]
 impl From<ParseSyntaxError> for LoadingError {
     fn from(error: ParseSyntaxError) -> LoadingError {
@@ -136,6 +149,8 @@ impl Error for LoadingError {
             Io(ref error) => error.description(),
             #[cfg(feature = "yaml-load")]
             ParseSyntax(ref error, ..) => error.description(),
+            #[cfg(feature = "metadata")]
+            ParseMetadata(_) => "Failed to parse JSON",
             ParseTheme(_) => "Invalid syntax theme",
             ReadSettings(_) => "Invalid syntax theme settings",
             BadPath => "Invalid path",