Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Single pass ReplaceSource .node performance improvement #23

Merged
merged 7 commits into from
Aug 29, 2018
Merged

Single pass ReplaceSource .node performance improvement #23

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
b2cbbe4
Single pass ReplaceSource node
brentwilton Jul 21, 2017
6ac5075
Add unit test coverage for ReplaceSource
brentwilton Jul 30, 2017
dc35c1a
Merge branch 'master' into single-pass-replacesource-node
brentwilton Nov 27, 2017
d178f18
Fix double emit on in progress replace and adjust node column for ove…
brentwilton Dec 11, 2017
441ef8b
Move ReplaceSource replacement sort method outside of class
brentwilton Mar 2, 2018
89e8ebf
Simplify and inline methods
sokra Aug 29, 2018
cf11460
Improve the quality of the source maps in some cases
sokra Aug 29, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
240 changes: 180 additions & 60 deletions lib/ReplaceSource.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,30 +71,60 @@ class ReplaceSource extends Source {
}

node(options) {
this._sortReplacements();
var result = [this._source.node(options)];
this.replacements.forEach(function(repl) {
var remSource = result.pop();
var splitted1 = this._splitSourceNode(remSource, Math.floor(repl[1] + 1));
var splitted2;
if(Array.isArray(splitted1)) {
splitted2 = this._splitSourceNode(splitted1[0], Math.floor(repl[0]));
if(Array.isArray(splitted2)) {
result.push(splitted1[1], this._replacementToSourceNode(splitted2[1], repl[2]), splitted2[0]);
} else {
result.push(splitted1[1], this._replacementToSourceNode(splitted1[1], repl[2]), splitted1[0]);
}
} else {
splitted2 = this._splitSourceNode(remSource, Math.floor(repl[0]));
if(Array.isArray(splitted2)) {
result.push(this._replacementToSourceNode(splitted2[1], repl[2]), splitted2[0]);
} else {
result.push(repl[2], remSource);
}
var node = this._source.node(options);
if(this.replacements.length === 0) {
return node;
}
this.replacements.sort(sortReplacementsAscending);
var replace = new ReplacementEnumerator(this.replacements);
var output = [];
var position = 0;
var sources = Object.create(null);
var sourcesInLines = Object.create(null);

// We build a new list of SourceNodes in "output"
// from the original mapping data

var result = new SourceNode();

// We need to add source contents manually
// because "walk" will not handle it
node.walkSourceContents(function(sourceFile, sourceContent) {
result.setSourceContent(sourceFile, sourceContent);
sources["$" + sourceFile] = sourceContent;
});

var replaceInStringNode = this._replaceInStringNode.bind(this, output, replace, function getOriginalSource(mapping) {
var key = "$" + mapping.source;
var lines = sourcesInLines[key];
if(!lines) {
var source = sources[key];
if(!source) return null;
lines = source.split("\n").map(function(line) {
return line + "\n";
});
sourcesInLines[key] = lines;
}
}, this);
result = result.reverse();
return new SourceNode(null, null, null, result);
// line is 1-based
if(mapping.line > lines.length) return null;
var line = lines[mapping.line - 1];
return line.substr(mapping.column);
});

node.walk(function(chunk, mapping) {
position = replaceInStringNode(chunk, position, mapping);
});

// If any replacements occur after the end of the original file, then we append them
// directly to the end of the output
var remaining = replace.footer();
if(remaining) {
output.push(remaining);
}

result.add(output);

return result;
}

listMap(options) {
Expand Down Expand Up @@ -149,52 +179,142 @@ class ReplaceSource extends Source {
return map;
}

_replacementToSourceNode(oldNode, newString) {
var map = oldNode.toStringWithSourceMap({
file: "?"
}).map;
var original = new SourceMapConsumer(map.toJSON()).originalPositionFor({
line: 1,
column: 0
});
if(original) {
return new SourceNode(original.line, original.column, original.source, newString);
} else {
return newString;
}
_splitString(str, position) {
return position <= 0 ? ["", str] : [str.substr(0, position), str.substr(position)];
}

_splitSourceNode(node, position) {
if(typeof node === "string") {
if(node.length <= position) return position - node.length;
return position <= 0 ? ["", node] : [node.substr(0, position), node.substr(position)];
} else {
for(var i = 0; i < node.children.length; i++) {
position = this._splitSourceNode(node.children[i], position);
if(Array.isArray(position)) {
var leftNode = new SourceNode(
node.line,
node.column,
node.source,
node.children.slice(0, i).concat([position[0]]),
node.name
_replaceInStringNode(output, replace, getOriginalSource, node, position, mapping) {
var original = undefined;

do {
var splitPosition = replace.position - position;
// If multiple replaces occur in the same location then the splitPosition may be
// before the current position for the subsequent splits. Ensure it is >= 0
if(splitPosition < 0) {
splitPosition = 0;
}
if(splitPosition >= node.length || replace.done) {
if(replace.emit) {
var nodeEnd = new SourceNode(
mapping.line,
mapping.column,
mapping.source,
node,
mapping.name
);
var rightNode = new SourceNode(
node.line,
node.column,
node.source, [position[1]].concat(node.children.slice(i + 1)),
node.name
output.push(nodeEnd);
}
return position + node.length;
}

var originalColumn = mapping.column;

// Try to figure out if generated code matches original code of this segement
// If this is the case we assume that it's allowed to move mapping.column
// Because getOriginalSource can be expensive we only do it when neccessary

var nodePart;
if(splitPosition > 0) {
nodePart = node.slice(0, splitPosition);
if(original === undefined) {
original = getOriginalSource(mapping);
}
if(original && original.length >= splitPosition && original.startsWith(nodePart)) {
mapping.column += splitPosition;
original = original.substr(splitPosition);
}
}

var emit = replace.next();
if(!emit) {
// Stop emitting when we have found the beginning of the string to replace.
// Emit the part of the string before splitPosition
if(splitPosition > 0) {
var nodeStart = new SourceNode(
mapping.line,
originalColumn,
mapping.source,
nodePart,
mapping.name
);
leftNode.sourceContents = node.sourceContents;
return [leftNode, rightNode];
output.push(nodeStart);
}

// Emit the replacement value
if(replace.value) {
output.push(new SourceNode(
mapping.line,
mapping.column,
mapping.source,
replace.value
));
}
}

// Recurse with remainder of the string as there may be multiple replaces within a single node
node = node.substr(splitPosition);
position += splitPosition;
} while (true);
}
}

function sortReplacementsAscending(a, b) {
var diff = a[1] - b[1]; // end
if(diff !== 0)
return diff;
diff = a[0] - b[0]; // start
if(diff !== 0)
return diff;
return a[3] - b[3]; // insert order
}

class ReplacementEnumerator {
constructor(replacements) {
this.emit = true;
this.done = !replacements || replacements.length === 0;
this.index = 0;
this.replacements = replacements;
if(!this.done) {
// Set initial start position in case .header is not called
var repl = replacements[0];
this.position = Math.floor(repl[0]);
if(this.position < 0)
this.position = 0;
}
}

next() {
if(this.done)
return true;
if(this.emit) {
// Start point found. stop emitting. set position to find end
var repl = this.replacements[this.index];
var end = Math.floor(repl[1] + 1);
this.position = end;
this.value = repl[2];
} else {
// End point found. start emitting. set position to find next start
this.index++;
if(this.index >= this.replacements.length) {
this.done = true;
} else {
var nextRepl = this.replacements[this.index];
var start = Math.floor(nextRepl[0]);
this.position = start;
}
return position;
}
if(this.position < 0)
this.position = 0;
this.emit = !this.emit;
return this.emit;
}

_splitString(str, position) {
return position <= 0 ? ["", str] : [str.substr(0, position), str.substr(position)];
footer() {
if(!this.done && !this.emit)
this.next(); // If we finished _replaceInNode mid emit we advance to next entry
return this.done ? [] : this.replacements.slice(this.index).map(function(repl) {
return repl[2];
}).join("");
}
}

Expand Down
81 changes: 81 additions & 0 deletions test/ReplaceSource.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ describe("ReplaceSource", function() {
resultListMap.map.version.should.be.eql(resultMap.map.version);
resultListMap.map.sources.should.be.eql(resultMap.map.sources);
resultListMap.map.sourcesContent.should.be.eql(resultMap.map.sourcesContent);
resultMap.map.mappings.should.be.eql("AAAA,CAAC,EAAI,KAAE,IAAC;AACR,CAAC;AAAA;AAAA;AAID,IAAI,CACJ");
resultListMap.map.mappings.should.be.eql("AAAA;AACA;AAAA;AAAA;AAIA,KACA");
});

Expand All @@ -64,13 +65,93 @@ describe("ReplaceSource", function() {
var resultListMap = source.sourceAndMap({
columns: false
});

resultText.should.be.eql("Message: Hey Ad!");
resultMap.source.should.be.eql(resultText);
resultListMap.source.should.be.eql(resultText);
resultListMap.map.file.should.be.eql(resultMap.map.file);
resultListMap.map.version.should.be.eql(resultMap.map.version);
resultListMap.map.sources.should.be.eql(resultMap.map.sources);
resultListMap.map.sourcesContent.should.be.eql(resultMap.map.sourcesContent);
resultMap.map.mappings.should.be.eql("AAAA,WAAE,GACE");
resultListMap.map.mappings.should.be.eql("AAAA,cACA");
});

it("should prepend items correctly", function() {
var source = new ReplaceSource(
new OriginalSource("Line 1", "file.txt")
);
source.insert(-1, "Line -1\n");
source.insert(-1, "Line 0\n");
var resultText = source.source();
var resultMap = source.sourceAndMap({
columns: true
});
var resultListMap = source.sourceAndMap({
columns: false
});

resultText.should.be.eql("Line -1\nLine 0\nLine 1");
resultMap.source.should.be.eql(resultText);
resultListMap.source.should.be.eql(resultText);
resultListMap.map.file.should.be.eql(resultMap.map.file);
resultListMap.map.version.should.be.eql(resultMap.map.version);
resultListMap.map.sources.should.be.eql(resultMap.map.sources);
resultListMap.map.sourcesContent.should.be.eql(resultMap.map.sourcesContent);
resultMap.map.mappings.should.be.eql("AAAA;AAAA;AAAA");
resultListMap.map.mappings.should.be.eql("AAAA;AAAA;AAAA");
});

it("should prepend items with replace at start correctly", function() {
var source = new ReplaceSource(
new OriginalSource([
"Line 1",
"Line 2"
].join("\n"), "file.txt")
);
source.insert(-1, "Line 0\n");
source.replace(0, 5, "Hello");
var resultText = source.source();
var resultMap = source.sourceAndMap({
columns: true
});
var resultListMap = source.sourceAndMap({
columns: false
});

resultText.should.be.eql("Line 0\nHello\nLine 2");
resultMap.source.should.be.eql(resultText);
resultListMap.source.should.be.eql(resultText);
resultListMap.map.file.should.be.eql(resultMap.map.file);
resultListMap.map.version.should.be.eql(resultMap.map.version);
resultListMap.map.sources.should.be.eql(resultMap.map.sources);
resultListMap.map.sourcesContent.should.be.eql(resultMap.map.sourcesContent);
resultMap.map.mappings.should.be.eql("AAAA;AAAA,KAAM;AACN");
resultListMap.map.mappings.should.be.eql("AAAA;AAAA;AACA");
});

it("should append items correctly", function() {
var line1;
var source = new ReplaceSource(
new OriginalSource(line1 = "Line 1\n", "file.txt")
);
source.insert(line1.length + 1, "Line 2\n");
var resultText = source.source();
var resultMap = source.sourceAndMap({
columns: true
});
var resultListMap = source.sourceAndMap({
columns: false
});

resultText.should.be.eql("Line 1\nLine 2\n");
resultMap.source.should.be.eql(resultText);
resultListMap.source.should.be.eql(resultText);
resultListMap.map.file.should.be.eql(resultMap.map.file);
resultListMap.map.version.should.be.eql(resultMap.map.version);
resultListMap.map.sources.should.be.eql(resultMap.map.sources);
resultListMap.map.sourcesContent.should.be.eql(resultMap.map.sourcesContent);
resultMap.map.mappings.should.be.eql("AAAA");
resultListMap.map.mappings.should.be.eql("AAAA;;");
});
});