Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Stricter trivia selection. #1304

Merged
merged 3 commits into from
Dec 18, 2020
Merged

Stricter trivia selection. #1304

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f490b69
Don't consider numbers that only consist out of digits as trivia.
nojaf Dec 18, 2020
fb03da7
Don't consider simple chars as trivia.
nojaf Dec 18, 2020
79510eb
Only consider the special strings as trivia.
nojaf Dec 18, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 4 additions & 16 deletions src/Fantomas.Tests/TokenParserTests.fs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@ let ``Single line block comment should be found in tokens`` () =
| _ -> failwith "expected block comment"

[<Test>]
let ``Multi line block comment should be found in tokens`` () =
let ``multi line block comment should be found in tokens`` () =
let source = """let bar =
(* multi
line
Expand All @@ -179,7 +179,7 @@ let ``Multi line block comment should be found in tokens`` () =

match triviaNodes with
| [ { Item = Comment (BlockComment (blockComment, _, _))
Range = range }; { Item = Number ("7") } ] ->
Range = range } ] ->
blockComment == expectedComment
range.StartLine == 2
range.EndLine == 4
Expand Down Expand Up @@ -241,7 +241,7 @@ let ``Comment after left brace of record`` () =

match triviaNodes with
| [ { Item = Comment (LineCommentAfterSourceCode (comment))
Range = range }; { Item = Number ("7") } ] ->
Range = range } ] ->
comment == "// foo"
range.StartLine == 2
| _ -> failwith "expected line comment after left brace"
Expand All @@ -267,7 +267,7 @@ type T() =
let triviaNodes = tokenize source |> getTriviaFromTokens

match triviaNodes with
| [ { Item = Newline; Range = rAbove }; { Item = Number ("123") } ] -> rAbove.StartLine == 1
| [ { Item = Newline; Range = rAbove } ] -> rAbove.StartLine == 1
| _ -> fail ()

[<Test>]
Expand Down Expand Up @@ -403,7 +403,6 @@ let ``with quotes`` () =

List.length triviaNodes == 1


[<Test>]
let ``infix operator in full words inside an ident`` () =
let source = """let op_LessThan(a, b) = a < b"""
Expand All @@ -429,16 +428,6 @@ let ``ident between tickets `` () =
| [ { Item = IdentBetweenTicks ("``/ operator combines paths``") } ] -> pass ()
| _ -> fail ()

[<Test>]
let ``simple char content`` () =
let source = "let someChar = \'s\'"

let triviaNodes = tokenize source |> getTriviaFromTokens

match triviaNodes with
| [ { Item = CharContent ("\'s\'") } ] -> pass ()
| _ -> fail ()

[<Test>]
let ``escaped char content`` () =
let source = "let nulchar = \'\\u0000\'"
Expand Down Expand Up @@ -507,7 +496,6 @@ let a = \"\\\"

getDefines source == []


[<Test>]
let ``defines inside triple quote string`` () =
let source = "
Expand Down
38 changes: 20 additions & 18 deletions src/Fantomas.Tests/TriviaTests.fs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,7 @@ let a = 9
let triviaNodes = toTrivia source |> List.head

match triviaNodes with
| [ { ContentBefore = [ Comment (LineCommentOnSingleLine (lineComment)) ] }; { ContentItself = Some (Number ("9")) } ] ->
lineComment == "// meh"
| [ { ContentBefore = [ Comment (LineCommentOnSingleLine (lineComment)) ] } ] -> lineComment == "// meh"
| _ -> failwith "Expected line comment"

[<Test>]
Expand All @@ -50,8 +49,7 @@ let a = 'c'
let triviaNodes = toTrivia source |> List.head

match triviaNodes with
| [ { ContentBefore = [ Comment (LineCommentOnSingleLine (lineComment)) ] };
{ ContentItself = Some (CharContent ("\'c\'")) } ] -> lineComment == "// foo"
| [ { ContentBefore = [ Comment (LineCommentOnSingleLine (lineComment)) ] } ] -> lineComment == "// foo"
| _ -> failwith "Expected line comment"

[<Test>]
Expand All @@ -61,9 +59,7 @@ let ``line comment on same line, is after last AST item`` () =

match triviaNodes with
| [ { Type = MainNode (SynModuleOrNamespace_AnonModule)
ContentAfter = [ Comment (LineCommentAfterSourceCode (lineComment)) ] };
{ Type = MainNode (SynExpr_Const)
ContentItself = Some (Number ("7")) } ] -> lineComment == "// should be 8"
ContentAfter = [ Comment (LineCommentAfterSourceCode (lineComment)) ] } ] -> lineComment == "// should be 8"
| _ -> fail ()

[<Test>]
Expand All @@ -74,8 +70,7 @@ let b = 9"""
let triviaNodes = toTrivia source |> List.head

match triviaNodes with
| [ { ContentItself = Some (Number ("7")) }; { ContentBefore = [ Newline ] };
{ ContentItself = Some (Number ("9")) } ] -> pass ()
| [ { ContentBefore = [ Newline ] } ] -> pass ()
| _ -> fail ()

[<Test>]
Expand All @@ -93,7 +88,7 @@ let a = 7
// bar"""

match triviaNodes with
| [ { ContentBefore = [ Comment (LineCommentOnSingleLine (comments)) ] }; { ContentItself = Some (Number ("7")) } ] ->
| [ { ContentBefore = [ Comment (LineCommentOnSingleLine (comments)) ] } ] ->
String.normalizeNewLine comments
== expectedComment
| _ -> fail ()
Expand All @@ -109,8 +104,7 @@ let ``comments inside record`` () =

match triviaNodes with
| [ { Type = TriviaNodeType.Token (LBRACE, _)
ContentAfter = [ Comment (LineCommentAfterSourceCode ("// foo")) ] }; { ContentItself = Some (Number ("7")) } ] ->
pass ()
ContentAfter = [ Comment (LineCommentAfterSourceCode ("// foo")) ] } ] -> pass ()
| _ -> fail ()

[<Test>]
Expand All @@ -124,8 +118,7 @@ let ``comment after all source code`` () =

match triviaNodes with
| [ { Type = MainNode (mn)
ContentAfter = [ Comment (LineCommentOnSingleLine (lineComment)) ] };
{ ContentItself = Some (Number ("123")) } ] ->
ContentAfter = [ Comment (LineCommentOnSingleLine (lineComment)) ] } ] ->
mn == SynModuleDecl_Types

lineComment
Expand All @@ -141,8 +134,7 @@ let ``block comment added to trivia`` () =
let triviaNodes = toTrivia source |> List.head

match triviaNodes with
| [ { ContentBefore = [ Comment (BlockComment (comment, _, _)) ]
ContentItself = Some (Number ("9")) } ] -> comment == "(* meh *)"
| [ { ContentBefore = [ Comment (BlockComment (comment, _, _)) ] } ] -> comment == "(* meh *)"
| _ -> failwith "Expected block comment"

[<Test>]
Expand Down Expand Up @@ -200,8 +192,7 @@ let a = 9
let triviaNodes = toTrivia source |> List.head

match triviaNodes with
| [ { ContentBefore = [ Comment (BlockComment (comment, _, true)) ] }; { ContentItself = Some (Number ("9")) } ] ->
comment == "(* // meh *)"
| [ { ContentBefore = [ Comment (BlockComment (comment, _, true)) ] } ] -> comment == "(* // meh *)"
| _ -> failwith "Expected block comment"


Expand Down Expand Up @@ -487,3 +478,14 @@ type LongIdentWithDots =
| [ { ContentBefore = [ Comment (LineCommentOnSingleLine (comment)) ] } ] ->
String.normalizeNewLine comment == expectedComment
| _ -> fail ()

[<Test>]
let ``number expression`` () =
let source = sprintf "let x = 2.0m"

let trivia = toTrivia source |> List.head

match trivia with
| [ { ContentItself = Some (Number (n))
Type = TriviaNodeType.MainNode (SynExpr_Const) } ] -> n == "2.0m"
| _ -> fail ()
135 changes: 107 additions & 28 deletions src/Fantomas/TokenParser.fs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -487,9 +487,103 @@ let private isOperatorOrKeyword ({ TokenInfo = { CharClass = cc } }) =
cc = FSharpTokenCharKind.Keyword
|| cc = FSharpTokenCharKind.Operator

let private isNumber ({ TokenInfo = tn }) =
let private onlyNumberRegex =
System.Text.RegularExpressions.Regex(@"^\d+$")

let private isNumber ({ TokenInfo = tn; Content = content }) =
tn.ColorClass = FSharpTokenColorKind.Number
&& List.contains tn.TokenName numberTrivia
&& not (onlyNumberRegex.IsMatch(content))

let private digitOrLetterCharRegex =
System.Text.RegularExpressions.Regex(@"^'(\d|[a-zA-Z])'$")

let private (|CharToken|_|) token =
if token.TokenInfo.TokenName = "CHAR"
&& not (digitOrLetterCharRegex.IsMatch(token.Content)) then
Some token
else
None

let private (|StringTextToken|_|) token =
if token.TokenInfo.TokenName = "STRING_TEXT" then
Some token
else
None

let private (|InterpStringEndToken|_|) token =
if token.TokenInfo.TokenName = "INTERP_STRING_END" then
Some token
else
None

let escapedCharacterRegex =
System.Text.RegularExpressions.Regex("(\\\\(n|r|u|\\\"|\\\\))+")

let rec private (|EndOfInterpolatedString|_|) tokens =
match tokens with
| StringTextToken (stToken) :: InterpStringEndToken (endToken) :: rest -> Some([ stToken ], endToken, rest)
| StringTextToken (stToken) :: EndOfInterpolatedString (stringTokens, endToken, rest) ->
Some(stToken :: stringTokens, endToken, rest)
| _ -> None

let private (|StringText|_|) tokens =
match tokens with
| StringTextToken head :: rest ->
let stringTokens =
rest
|> List.takeWhile (fun { TokenInfo = { TokenName = tn } } -> tn = "STRING_TEXT")
|> fun others ->
let length = List.length others
let closingQuote = rest.[length]

[ yield head
yield! others
yield closingQuote ]

let stringContent =
let builder = StringBuilder()

stringTokens
|> List.fold
(fun (b: StringBuilder, currentLine) st ->
if currentLine <> st.LineNumber then
let delta = st.LineNumber - currentLine

[ 1 .. delta ]
|> List.iter (fun _ -> b.Append("\n") |> ignore)

b.Append(st.Content), st.LineNumber
else
b.Append(st.Content), st.LineNumber)
(builder, head.LineNumber)
|> fst
|> fun b -> b.ToString()

let stringStartIsSpecial () =
if stringContent.Length > 2 then
match stringContent.[0], stringContent.[1], stringContent.[2] with
| '@', '"', _
| '$', '"', _
| '"', '"', '"' -> true
| _ -> false
else
false

let hasEscapedCharacter () =
escapedCharacterRegex.IsMatch(stringContent)

let hasNewlines () = stringContent.Contains("\n")
let endsWithBinaryCharacter () = stringContent.EndsWith("\"B")

if stringStartIsSpecial ()
|| hasEscapedCharacter ()
|| hasNewlines ()
|| endsWithBinaryCharacter () then
Some(head, stringTokens, rest, stringContent)
else
None
| _ -> None

let private identIsDecompiledOperator (token: Token) =
let decompiledName =
Expand Down Expand Up @@ -643,37 +737,22 @@ let rec private getTriviaFromTokensThemSelves (allTokens: Token list) (tokens: T

getTriviaFromTokensThemSelves allTokens nextRest info

| head :: rest when (head.TokenInfo.TokenName = "STRING_TEXT") ->
let stringTokens =
rest
|> List.takeWhile (fun { TokenInfo = { TokenName = tn } } -> tn = "STRING_TEXT")
|> fun others ->
let length = List.length others
let closingQuote = rest.[length]

[ yield head
yield! others
yield closingQuote ]

| EndOfInterpolatedString (stringTokens, interpStringEnd, rest) ->
let stringContent =
let builder = StringBuilder()
[ yield! (List.map (fun t -> t.Content) stringTokens)
yield interpStringEnd.Content ]
|> String.concat String.Empty

stringTokens
|> List.fold
(fun (b: StringBuilder, currentLine) st ->
if currentLine <> st.LineNumber then
let delta = st.LineNumber - currentLine
let range =
getRangeBetween "string content" stringTokens.Head interpStringEnd

[ 1 .. delta ]
|> List.iter (fun _ -> b.Append("\n") |> ignore)
let info =
Trivia.Create(StringContent(stringContent)) range
|> List.prependItem foundTrivia

b.Append(st.Content), st.LineNumber
else
b.Append(st.Content), st.LineNumber)
(builder, head.LineNumber)
|> fst
|> fun b -> b.ToString()
getTriviaFromTokensThemSelves allTokens rest info

| StringText (head, stringTokens, rest, stringContent) ->
let lastToken =
List.tryLast stringTokens
|> Option.defaultValue head
Expand Down Expand Up @@ -733,7 +812,7 @@ let rec private getTriviaFromTokensThemSelves (allTokens: Token list) (tokens: T

getTriviaFromTokensThemSelves allTokens rest info

| head :: rest when (head.TokenInfo.TokenName = "CHAR") ->
| CharToken (head) :: rest ->
let range =
getRangeBetween head.TokenInfo.TokenName head head

Expand Down