Skip to content

Commit

Permalink
Fix line and column number reporting for Windows newlines (#282)
Browse files Browse the repository at this point in the history
Previous implementation of `lineNumberLookup` only worked for single
`\n` or `\r` newlines. This PR fixes it so it also works for `\n\r` or
`\r\n` newlines and adds some basic tests
  • Loading branch information
lihaoyi authored Mar 24, 2023
1 parent 186337a commit 6f5875e
Show file tree
Hide file tree
Showing 2 changed files with 66 additions and 14 deletions.
29 changes: 15 additions & 14 deletions fastparse/src/fastparse/internal/Util.scala
Original file line number Diff line number Diff line change
Expand Up @@ -47,30 +47,31 @@ object Util {
rec(0)
}
def lineNumberLookup(data: String): Array[Int] = {
val lineStarts = new ArrayBuffer[Int]()
val lineStarts = ArrayBuffer[Int](0)
var i = 0
var col = 1
var cr = false
var prev: Character = null
// Stores the previous char we saw, or -1 if we just saw a \r\n or \n\r pair
var state: Int = 0
while (i < data.length){
val char = data(i)
if (char == '\r') {
if (prev != '\n' && col == 1) lineStarts.append(i)
col = 1
cr = true
}else if (char == '\n') {
if (prev != '\r' && col == 1) lineStarts.append(i)
if (char == '\r' && state == '\n' || char == '\n' && state == '\r'){
col += 1
state = -1
} else if (state == '\r' || state == '\n' || state == -1) {
lineStarts.append(i)
col = 1
cr = false
state = char
}else{
if (col == 1) lineStarts.append(i)
col += 1
cr = false
state = char
}
prev = char

i += 1
}
if (col == 1) lineStarts.append(i)

if (state == '\r' || state == '\n' || state == -1) {
lineStarts.append(i)
}

lineStarts.toArray
}
Expand Down
51 changes: 51 additions & 0 deletions fastparse/test/src/fastparse/UtilTests.scala
Original file line number Diff line number Diff line change
Expand Up @@ -42,5 +42,56 @@ object UtilTests extends TestSuite {
)
assert(pretties == expected)
}

test("unix"){
val txt = Array(
"def myScalaVersion = \"2.13.2\"\n",
"\n",
"//hello\n",
"println(doesntExis})"
).mkString

val lineStarts = fastparse.internal.Util.lineNumberLookup(txt).toList

assert(lineStarts == List(0, 30, 31, 39))
}

test("carriageReturnOnly") {
val txt = Array(
"def myScalaVersion = \"2.13.2\"\r",
"\r",
"//hello\r",
"println(doesntExis})"
).mkString

val lineStarts = fastparse.internal.Util.lineNumberLookup(txt).toList

assert(lineStarts == List(0, 30, 31, 39))
}

test("windows"){
val txt = Array(
"def myScalaVersion = \"2.13.2\"\r\n",
"\r\n",
"//hello\r\n",
"println(doesntExis})"
).mkString

val lineStarts = fastparse.internal.Util.lineNumberLookup(txt).toList

assert(lineStarts == List(0, 31, 33, 42))
}
test("reverseWindows"){
val txt = Array(
"def myScalaVersion = \"2.13.2\"\n\r",
"\n\r",
"//hello\n\r",
"println(doesntExis})"
).mkString

val lineStarts = fastparse.internal.Util.lineNumberLookup(txt).toList

assert(lineStarts == List(0, 31, 33, 42))
}
}
}

0 comments on commit 6f5875e

Please sign in to comment.