Merge branch 'master' into syntest

Author: keith-hall

.travis.yml

@@ -25,13 +25,6 @@ cache:
 
 script:
   - cargo build
-  - |
-    if [ "$TRAVIS_OS_NAME" == "linux" ]
-    then
-      p=$(cd ./target/debug/build/onig_sys-*/out/lib/ && pwd)
-      echo "adding $p to linker path"
-      export LD_LIBRARY_PATH="${p}:${LD_LIBRARY_PATH}"
-    fi
   - cargo test
   - make assets
   - make syntest

CHANGELOG.md

@@ -0,0 +1,54 @@
+# Version 3.0
+
+This is a major release with multiple breaking API changes, although upgrading shouldn't be too difficult. It fixes bugs and comes with some nice new features.
+
+## Breaking changes and upgrading
+
+- The `SyntaxSet` API has been revamped to use a builder and an arena of contexts. See [example usage](https://github.com/trishume/syntect/blob/51208d35a6d98c07468fbe044d5c6f37eb129205/examples/gendata.rs#L25-L28).
+- Many functions now need to be passed the `SyntaxSet` that goes with the rest of their arguments because of this new arena.
+- Filename added to `LoadingError::ParseSyntax`
+- Many functions in the `html` module now take the `newlines` version of syntaxes.
+  - These methods have also been renamed, partially so that code that needs updating doesn't break without a compile error.
+  - The HTML they output also treats newlines slightly differently and I think more correctly but uglier when you look at the HTML.
+
+#### Breaking rename upgrade guide
+
+- `SyntaxSet::add_syntax -> SyntaxSetBuilder::add`
+- `SyntaxSet::load_syntaxes -> SyntaxSetBuilder::add_from_folder`
+- `SyntaxSet::load_plain_text_syntax -> SyntaxSetBuilder::add_plain_text_syntax`
+- `html::highlighted_snippet_for_string -> html::highlighted_html_for_string`: also change to `newlines` `SyntaxSet`
+- `html::highlighted_snippet_for_file -> html::highlighted_html_for_file`: also change to `newlines` `SyntaxSet`
+- `html::styles_to_coloured_html -> html::styled_line_to_highlighted_html`: also change to `newlines` `SyntaxSet`
+- `html::start_coloured_html_snippet -> html::start_highlighted_html_snippet`: return type also changed
+
+## Major changes and new features
+
+- Use arena for contexts (#182 #186 #187 #190 #195): This makes the code cleaner, enables use of syntaxes from multiple threads, and prevents accidental misuse.
+  - This involves a new `SyntaxSetBuilder` API for constructing new `SyntaxSet`s
+  - See the revamped [parsyncat example](https://github.com/trishume/syntect/blob/51208d35a6d98c07468fbe044d5c6f37eb129205/examples/parsyncat.rs).
+- Encourage use of newlines (#197 #207 #196): The `nonewlines` mode is often buggy so we made it easier to use the `newlines` mode.
+  - Added a `LinesWithEndings` utility for iterating over the lines of a string with `\n` characters.
+  - Reengineer the `html` module to use `newlines` syntaxes.
+- Add helpers for modifying highlighted lines (#198): For use cases like highlighting a piece of text in a blog code snippet or debugger. This allows you to reach into the highlighted spans and add styles.
+  - Check out `split_at` and `modify_range` in the `util` module.
+- New `ThemeSet::add_from_folder` function (#200): For modifying existing theme sets.
+
+## Bug Fixes
+
+- Improve nonewlines regex rewriting: #212 #211
+- Reengineer theme application to match Sublime: #209
+- Also mark contexts referenced by name as "no prototype" (same as ST): #180
+- keep with_prototype when switching contexts with `set`: #177 #166
+- Fix unused import warning: #174
+- Ignore trailing dots in selectors: #173
+- Fix `embed` to not include prototypes: #172 #160
+
+## Upgraded dependencies
+
+- plist: `0.2 -> 0.3`
+- regex: `0.2 -> 1.0`
+- onig: `3.2.1 -> 4.1`
+
+# Prior versions
+
+See the Github release notes: <https://github.com/trishume/syntect/releases>

Cargo.toml

@@ -7,31 +7,32 @@ keywords = ["syntax", "highlighting", "highlighter", "colouring", "parsing"]
 categories = ["parser-implementations", "parsing", "text-processing"]
 readme = "Readme.md"
 license = "MIT"
-version = "2.1.0" # remember to update html_root_url
+version = "3.0.0" # remember to update html_root_url
 authors = ["Tristan Hume <[email protected]>"]
 exclude = [
     "testdata/*",
 ]
 
 [dependencies]
 yaml-rust = { version = "0.4", optional = true }
-onig = { version = "3.2.1", optional = true }
+onig = { version = "4.1", optional = true }
 walkdir = "2.0"
-regex-syntax = { version = "0.4", optional = true }
+regex-syntax = { version = "0.6", optional = true }
 lazy_static = "1.0"
+lazycell = "1.0"
 bitflags = "1.0"
 plist = "0.3"
 bincode = { version = "1.0", optional = true }
 flate2 = { version = "1.0", optional = true, default-features = false }
 fnv = { version = "1.0", optional = true }
-serde = { version = "1.0", features = ["rc"] }
+serde = "1.0"
 serde_derive = "1.0"
 serde_json = "1.0"
 
 [dev-dependencies]
 criterion = "0.2"
 rayon = "1.0.0"
-regex = "0.2"
+regex = "1.0"
 getopts = "0.2"
 pretty_assertions = "0.5.0"
 

Readme.md

@@ -7,27 +7,23 @@
 
 `syntect` is a syntax highlighting library for Rust that uses [Sublime Text syntax definitions](http://www.sublimetext.com/docs/3/syntax.html#include-syntax). It aims to be a good solution for any Rust project that needs syntax highlighting, including deep integration with text editors written in Rust. It's used in production by at least two companies, and by [many open source projects](#projects-using-syntect).
 
-If you are writing a text editor (or something else needing highlighting) in Rust and this library doesn't fit your needs, I consider that a bug and you should file an issue or email me.
+If you are writing a text editor (or something else needing highlighting) in Rust and this library doesn't fit your needs, I consider that a bug and you should file an issue or email me. I consider this project mostly complete, I still maintain it and review PRs, but it's not under heavy development.
 
-**Note:** I consider this project "done" in the sense that it works quite well for its intended purpose, accomplishes the major goals I had, and I'm unlikely to make any sweeping changes.
-I won't be committing much anymore because the marginal return on additional work isn't very high. Rest assured if you submit PRs I will review them and likely merge promptly.
-I'll also quite possibly still fix issues and definitely offer advice and knowledge on how the library works. Basically I'll be maintaining the library but not developing it further.
-I've spent months working on, tweaking, optimizing, documenting and testing this library. If you still have any reasons you don't think it fits your needs, file an issue or email me.
+## Important Links
 
-### Rendered docs: <https://docs.rs/syntect>
+- API docs with examples: <https://docs.rs/syntect>
+- [Changelogs and upgrade notes for past releases](https://github.com/trishume/syntect/releases)
 
 ## Getting Started
 
 `syntect` is [available on crates.io](https://crates.io/crates/syntect). You can install it by adding this line to your `Cargo.toml`:
 
 ```toml
-syntect = "2.1"
+syntect = "3.0"
 ```
 
 After that take a look at the [documentation](https://docs.rs/syntect) and the [examples](https://github.com/trishume/syntect/tree/master/examples).
 
-**Note:** with stable Rust on Linux there is a possibility you might have to add `./target/debug/build/onig_sys-*/out/lib/` to your `LD_LIBRARY_PATH` environment variable. I dunno why or even if this happens on other places than Travis, but see `travis.yml` for what it does to make it work. Do this if you see `libonig.so: cannot open shared object file`.
-
 If you've cloned this repository, be sure to run
 
 ```
@@ -36,17 +32,10 @@ git submodule update --init
 
 to fetch all the required dependencies for running the tests.
 
-### Feature Flags
-
-Syntect makes heavy use of [cargo features](http://doc.crates.io/manifest.html#the-features-section), to support users who require only a subset of functionality. In particular, it is possible to use the highlighting component of syntect without the parser (for instance when hand-rolling a higher performance parser for a particular language), by adding `default-features = false` to the syntect entry in your `Cargo.toml`.
-
-For more information on available features, see the features section in `Cargo.toml`.
-
 ## Features/Goals
 
 - [x] Work with many languages (accomplished through using existing grammar formats)
-- [x] Highlight super quickly, faster than every editor except Sublime Text 3
-- [x] Load up quickly, currently in around 23ms but could potentially be even faster.
+- [x] Highlight super quickly, faster than nearly all text editors
 - [x] Include easy to use API for basic cases
 - [x] API allows use in fancy text editors with piece tables and incremental re-highlighting and the like.
 - [x] Expose internals of the parsing process so text editors can do things like cache parse states and use semantic info for code intelligence
@@ -55,6 +44,7 @@ For more information on available features, see the features section in `Cargo.t
 - [x] Well documented, I've tried to add a useful documentation comment to everything that isn't utterly self explanatory.
 - [x] Built-in output to coloured HTML `<pre>` tags or 24-bit colour ANSI terminal escape sequences.
 - [x] Nearly complete compatibility with Sublime Text 3, including lots of edge cases. Passes nearly all of Sublime's syntax tests, see [issue 59](https://github.com/trishume/syntect/issues/59).
+- [x] Load up quickly, currently in around 23ms but could potentially be even faster.
 
 ## Screenshots
 
@@ -65,22 +55,29 @@ There's currently an example program called `syncat` that prints one of the sour
 ![Solarized Light](http://i.imgur.com/l3zcO4J.png)
 ![InspiredGithub](http://i.imgur.com/a7U1r2j.png)
 
-## Roadmap
-
-- [x] Sketch out representation of a Sublime Text syntax
-- [x] Parse `.sublime-syntax` files into the representation.
-- [x] Write an interpreter for the `.sublime-syntax` state machine that highlights an incoming iterator of file lines into an iterator of scope-annotated text.
-- [x] Parse TextMate/Sublime Text theme files
-- [x] Highlight a scope-annotated iterator into a colour-annotated iterator for display.
-- [x] Ability to dump loaded packages as binary file and load them with lazy regex compilation for fast start up times.
-- [x] Bundle dumped default syntaxes into the library binary so library users don't need an assets folder with Sublime Text packages.
-- [x] Add nice API wrappers for simple use cases. The base APIs are designed for deep high performance integration with arbitrary text editor data structures.
-- [x] Document the API better and make things private that don't need to be public
-- [x] Detect file syntax based on first line
-- [x] Make it really fast (mosty two hot-paths need caching, same places Textmate 2 caches)
-- [ ] Make syncat a better demo, and maybe more demo programs
-- [ ] Add sRGB colour correction (not sure if this is necessary, could be the job of the text editor)
-- [ ] Add C bindings so it can be used as a C library from other languages.
+## Example Code
+
+Prints highlighted lines of a string to the terminal. See the [easy](https://docs.rs/syntect/latest/syntect/easy/index.html) and [html](https://docs.rs/syntect/latest/syntect/html/index.html) module docs for more basic use case examples.
+
+```rust
+use syntect::easy::HighlightLines;
+use syntect::parsing::SyntaxSet;
+use syntect::highlighting::{ThemeSet, Style};
+use syntect::util::{as_24_bit_terminal_escaped, LinesWithEndings};
+
+// Load these once at the start of your program
+let ps = SyntaxSet::load_defaults_newlines();
+let ts = ThemeSet::load_defaults();
+
+let syntax = ps.find_syntax_by_extension("rs").unwrap();
+let mut h = HighlightLines::new(syntax, &ts.themes["base16-ocean.dark"]);
+let s = "pub struct Wow { hi: u64 }\nfn blah() -> u64 {}";
+for line in LinesWithEndings::from(s) {
+    let ranges: Vec<(Style, &str)> = h.highlight(line, &ps);
+    let escaped = as_24_bit_terminal_escaped(&ranges[..], true);
+    println!("{}", escaped);
+}
+```
 
 ## Performance
 
@@ -110,7 +107,13 @@ All measurements were taken on a mid 2012 15" retina Macbook Pro.
 - ~1.9ms to parse and highlight the 30 line 791 character `testdata/highlight_test.erb` file. This works out to around 16,000 lines/second or 422 kilobytes/second.
 - ~250ms end to end for `syncat` to start, load the definitions, highlight the test file and shut down. This is mostly spent loading.
 
-### Caching
+## Feature Flags
+
+Syntect makes heavy use of [cargo features](http://doc.crates.io/manifest.html#the-features-section), to support users who require only a subset of functionality. In particular, it is possible to use the highlighting component of syntect without the parser (for instance when hand-rolling a higher performance parser for a particular language), by adding `default-features = false` to the syntect entry in your `Cargo.toml`.
+
+For more information on available features, see the features section in `Cargo.toml`.
+
+## Caching
 
 Because `syntect`'s API exposes internal cacheable data structures, there is a caching strategy that text editors can use that allows the text on screen to be re-rendered instantaneously regardless of the file size when a change is made after the initial highlight.
 
@@ -120,11 +123,15 @@ This way from the time the edit happens to the time the new colouring gets rende
 
 Any time the file is changed the latest cached state is found, the cache is cleared after that point, and a background job is started. Any already running jobs are stopped because they would be working on old state. This way you can just have one thread dedicated to highlighting that is always doing the most up-to-date work, or sleeping.
 
-### Parallelizing
+## Parallelizing
+
+Since 3.0, `syntect` can be used to do parsing/highlighting in parallel. `SyntaxSet` is both `Send` and `Sync` and so can easily be used from multiple threads. It is also `Clone`, which means you can construct a syntax set and then clone it to use for other threads if you prefer.
 
-`syntect` doesn't provide any built-in facilities to enable highlighting in parallel. Some of the important data structures are not thread-safe, either, most notably `SyntaxSet`. However, if you find yourself in need of highlighting lots of files in parallel, the recommendation is to use some sort of thread pooling, along with the `thread_local!` macro from `libstd`, so that each thread that needs, say, a `SyntaxSet`, will have one, while minimizing the amount of them that need to be initialized. For adding parallelism to a previously single-threaded program, the recommended thread pooling is [`rayon`](https://github.com/nikomatsakis/rayon). However, if you're working in an already-threaded context where there might be more threads than you want (such as writing a handler for an Iron request), the recommendation is to force all highlighting to be done within a fixed-size thread pool using [`rust-scoped-pool`](https://github.com/reem/rust-scoped-pool). An example of the former is in `examples/parsyncat.rs`.
+Compared to older versions, there's nothing preventing the serialization of a `SyntaxSet` either. So you can directly deserialize a fully linked `SyntaxSet` and start using it for parsing/highlighting. Before, it was always necessary to do linking first.
 
-See [#20](https://github.com/trishume/syntect/issues/20) and [#78](https://github.com/trishume/syntect/pull/78) for more detail and discussion about why `syntect` doesn't provide parallelism by default.
+It is worth mentioning that regex compilation is done lazily only when the regexes are actually needed. Once a regex has been compiled, the compiled version is used for all threads after that. Note that this is done using interior mutability, so if multiple threads happen to encounter the same uncompiled regex at the same time, compiling might happen multiple times. After that, one of the compiled regexes will be used. When a `SyntaxSet` is cloned, the regexes in the cloned set will need to be recompiled currently.
+
+For adding parallelism to a previously single-threaded program, the recommended thread pooling is [`rayon`](https://github.com/nikomatsakis/rayon). However, if you're working in an already-threaded context where there might be more threads than you want (such as writing a handler for an Iron request), the recommendation is to force all highlighting to be done within a fixed-size thread pool using [`rust-scoped-pool`](https://github.com/reem/rust-scoped-pool). An example of the former is in `examples/parsyncat.rs`.
 
 ## Examples Available
 
@@ -173,4 +180,6 @@ Below is a list of projects using Syntect, in approximate order by how long they
 
 ## License and Acknowledgements
 
+Thanks to [Robin Stocker](https://github.com/robinst) and also [Keith Hall](https://github.com/keith-hall) for making awesome substantial contributions of the most important impressive improvements `syntect` has had post-`v1.0`! They deserve lots of credit for where `syntect` is today.
+
 Thanks to [Textmate 2](https://github.com/textmate/textmate) and @defuz's [sublimate](https://github.com/defuz/sublimate) for the existing open source code I used as inspiration and in the case of sublimate's `tmTheme` loader, copy-pasted. All code (including defuz's sublimate code) is released under the MIT license.

assets/default_newlines.packdump

@@ -4,18 +4,19 @@ extern crate syntect;
 
 use criterion::{Bencher, Criterion};
 
-use syntect::parsing::{SyntaxSet, SyntaxDefinition, ScopeStack};
+use syntect::parsing::{SyntaxSet, SyntaxReference, ScopeStack};
 use syntect::highlighting::{ThemeSet, Theme};
 use syntect::easy::HighlightLines;
+use syntect::html::highlighted_html_for_string;
 use std::str::FromStr;
 use std::fs::File;
 use std::io::Read;
 
-fn do_highlight(s: &str, syntax: &SyntaxDefinition, theme: &Theme) -> usize {
+fn do_highlight(s: &str, syntax_set: &SyntaxSet, syntax: &SyntaxReference, theme: &Theme) -> usize {
     let mut h = HighlightLines::new(syntax, theme);
     let mut count = 0;
     for line in s.lines() {
-        let regions = h.highlight(line);
+        let regions = h.highlight(line, syntax_set);
         count += regions.len();
     }
     count
@@ -33,16 +34,16 @@ fn highlight_file(b: &mut Bencher, file: &str) {
     };
 
     // don't load from dump so we don't count lazy regex compilation time
-    let ps = SyntaxSet::load_defaults_nonewlines();
+    let ss = SyntaxSet::load_defaults_nonewlines();
     let ts = ThemeSet::load_defaults();
 
-    let syntax = ps.find_syntax_for_file(path).unwrap().unwrap();
+    let syntax = ss.find_syntax_for_file(path).unwrap().unwrap();
     let mut f = File::open(path).unwrap();
     let mut s = String::new();
     f.read_to_string(&mut s).unwrap();
 
     b.iter(|| {
-        do_highlight(&s, syntax, &ts.themes["base16-ocean.dark"])
+        do_highlight(&s, &ss, syntax, &ts.themes["base16-ocean.dark"])
     });
 }
 
@@ -56,8 +57,24 @@ fn stack_matching(b: &mut Bencher) {
     });
 }
 
+fn highlight_html(b: &mut Bencher) {
+    let ss = SyntaxSet::load_defaults_newlines();
+    let ts = ThemeSet::load_defaults();
+
+    let path = "testdata/parser.rs";
+    let syntax = ss.find_syntax_for_file(path).unwrap().unwrap();
+    let mut f = File::open(path).unwrap();
+    let mut s = String::new();
+    f.read_to_string(&mut s).unwrap();
+
+    b.iter(|| {
+        highlighted_html_for_string(&s, &ss, syntax, &ts.themes["base16-ocean.dark"])
+    });
+}
+
 fn highlighting_benchmark(c: &mut Criterion) {
     c.bench_function("stack_matching", stack_matching);
+    c.bench_function("highlight_html", highlight_html);
     c.bench_function_over_inputs(
         "highlight",
         |b, s| highlight_file(b, s),