diff --git a/src/Fantomas.Tests/TokenParserTests.fs b/src/Fantomas.Tests/TokenParserTests.fs index 08310eec46..7836c94645 100644 --- a/src/Fantomas.Tests/TokenParserTests.fs +++ b/src/Fantomas.Tests/TokenParserTests.fs @@ -162,7 +162,7 @@ let ``Single line block comment should be found in tokens`` () = | _ -> failwith "expected block comment" [] -let ``Multi line block comment should be found in tokens`` () = +let ``multi line block comment should be found in tokens`` () = let source = """let bar = (* multi line @@ -179,7 +179,7 @@ let ``Multi line block comment should be found in tokens`` () = match triviaNodes with | [ { Item = Comment (BlockComment (blockComment, _, _)) - Range = range }; { Item = Number ("7") } ] -> + Range = range } ] -> blockComment == expectedComment range.StartLine == 2 range.EndLine == 4 @@ -241,7 +241,7 @@ let ``Comment after left brace of record`` () = match triviaNodes with | [ { Item = Comment (LineCommentAfterSourceCode (comment)) - Range = range }; { Item = Number ("7") } ] -> + Range = range } ] -> comment == "// foo" range.StartLine == 2 | _ -> failwith "expected line comment after left brace" @@ -267,7 +267,7 @@ type T() = let triviaNodes = tokenize source |> getTriviaFromTokens match triviaNodes with - | [ { Item = Newline; Range = rAbove }; { Item = Number ("123") } ] -> rAbove.StartLine == 1 + | [ { Item = Newline; Range = rAbove } ] -> rAbove.StartLine == 1 | _ -> fail () [] @@ -403,7 +403,6 @@ let ``with quotes`` () = List.length triviaNodes == 1 - [] let ``infix operator in full words inside an ident`` () = let source = """let op_LessThan(a, b) = a < b""" @@ -429,16 +428,6 @@ let ``ident between tickets `` () = | [ { Item = IdentBetweenTicks ("``/ operator combines paths``") } ] -> pass () | _ -> fail () -[] -let ``simple char content`` () = - let source = "let someChar = \'s\'" - - let triviaNodes = tokenize source |> getTriviaFromTokens - - match triviaNodes with - | [ { Item = CharContent ("\'s\'") } ] -> pass () - | _ -> fail () - [] let ``escaped char content`` () = let source = "let nulchar = \'\\u0000\'" @@ -507,7 +496,6 @@ let a = \"\\\" getDefines source == [] - [] let ``defines inside triple quote string`` () = let source = " diff --git a/src/Fantomas.Tests/TriviaTests.fs b/src/Fantomas.Tests/TriviaTests.fs index 0ebf291d43..c697aed9b0 100644 --- a/src/Fantomas.Tests/TriviaTests.fs +++ b/src/Fantomas.Tests/TriviaTests.fs @@ -37,8 +37,7 @@ let a = 9 let triviaNodes = toTrivia source |> List.head match triviaNodes with - | [ { ContentBefore = [ Comment (LineCommentOnSingleLine (lineComment)) ] }; { ContentItself = Some (Number ("9")) } ] -> - lineComment == "// meh" + | [ { ContentBefore = [ Comment (LineCommentOnSingleLine (lineComment)) ] } ] -> lineComment == "// meh" | _ -> failwith "Expected line comment" [] @@ -50,8 +49,7 @@ let a = 'c' let triviaNodes = toTrivia source |> List.head match triviaNodes with - | [ { ContentBefore = [ Comment (LineCommentOnSingleLine (lineComment)) ] }; - { ContentItself = Some (CharContent ("\'c\'")) } ] -> lineComment == "// foo" + | [ { ContentBefore = [ Comment (LineCommentOnSingleLine (lineComment)) ] } ] -> lineComment == "// foo" | _ -> failwith "Expected line comment" [] @@ -61,9 +59,7 @@ let ``line comment on same line, is after last AST item`` () = match triviaNodes with | [ { Type = MainNode (SynModuleOrNamespace_AnonModule) - ContentAfter = [ Comment (LineCommentAfterSourceCode (lineComment)) ] }; - { Type = MainNode (SynExpr_Const) - ContentItself = Some (Number ("7")) } ] -> lineComment == "// should be 8" + ContentAfter = [ Comment (LineCommentAfterSourceCode (lineComment)) ] } ] -> lineComment == "// should be 8" | _ -> fail () [] @@ -74,8 +70,7 @@ let b = 9""" let triviaNodes = toTrivia source |> List.head match triviaNodes with - | [ { ContentItself = Some (Number ("7")) }; { ContentBefore = [ Newline ] }; - { ContentItself = Some (Number ("9")) } ] -> pass () + | [ { ContentBefore = [ Newline ] } ] -> pass () | _ -> fail () [] @@ -93,7 +88,7 @@ let a = 7 // bar""" match triviaNodes with - | [ { ContentBefore = [ Comment (LineCommentOnSingleLine (comments)) ] }; { ContentItself = Some (Number ("7")) } ] -> + | [ { ContentBefore = [ Comment (LineCommentOnSingleLine (comments)) ] } ] -> String.normalizeNewLine comments == expectedComment | _ -> fail () @@ -109,8 +104,7 @@ let ``comments inside record`` () = match triviaNodes with | [ { Type = TriviaNodeType.Token (LBRACE, _) - ContentAfter = [ Comment (LineCommentAfterSourceCode ("// foo")) ] }; { ContentItself = Some (Number ("7")) } ] -> - pass () + ContentAfter = [ Comment (LineCommentAfterSourceCode ("// foo")) ] } ] -> pass () | _ -> fail () [] @@ -124,8 +118,7 @@ let ``comment after all source code`` () = match triviaNodes with | [ { Type = MainNode (mn) - ContentAfter = [ Comment (LineCommentOnSingleLine (lineComment)) ] }; - { ContentItself = Some (Number ("123")) } ] -> + ContentAfter = [ Comment (LineCommentOnSingleLine (lineComment)) ] } ] -> mn == SynModuleDecl_Types lineComment @@ -141,8 +134,7 @@ let ``block comment added to trivia`` () = let triviaNodes = toTrivia source |> List.head match triviaNodes with - | [ { ContentBefore = [ Comment (BlockComment (comment, _, _)) ] - ContentItself = Some (Number ("9")) } ] -> comment == "(* meh *)" + | [ { ContentBefore = [ Comment (BlockComment (comment, _, _)) ] } ] -> comment == "(* meh *)" | _ -> failwith "Expected block comment" [] @@ -200,8 +192,7 @@ let a = 9 let triviaNodes = toTrivia source |> List.head match triviaNodes with - | [ { ContentBefore = [ Comment (BlockComment (comment, _, true)) ] }; { ContentItself = Some (Number ("9")) } ] -> - comment == "(* // meh *)" + | [ { ContentBefore = [ Comment (BlockComment (comment, _, true)) ] } ] -> comment == "(* // meh *)" | _ -> failwith "Expected block comment" @@ -487,3 +478,14 @@ type LongIdentWithDots = | [ { ContentBefore = [ Comment (LineCommentOnSingleLine (comment)) ] } ] -> String.normalizeNewLine comment == expectedComment | _ -> fail () + +[] +let ``number expression`` () = + let source = sprintf "let x = 2.0m" + + let trivia = toTrivia source |> List.head + + match trivia with + | [ { ContentItself = Some (Number (n)) + Type = TriviaNodeType.MainNode (SynExpr_Const) } ] -> n == "2.0m" + | _ -> fail () diff --git a/src/Fantomas/TokenParser.fs b/src/Fantomas/TokenParser.fs index fa4765cb90..3471fbf821 100644 --- a/src/Fantomas/TokenParser.fs +++ b/src/Fantomas/TokenParser.fs @@ -487,9 +487,103 @@ let private isOperatorOrKeyword ({ TokenInfo = { CharClass = cc } }) = cc = FSharpTokenCharKind.Keyword || cc = FSharpTokenCharKind.Operator -let private isNumber ({ TokenInfo = tn }) = +let private onlyNumberRegex = + System.Text.RegularExpressions.Regex(@"^\d+$") + +let private isNumber ({ TokenInfo = tn; Content = content }) = tn.ColorClass = FSharpTokenColorKind.Number && List.contains tn.TokenName numberTrivia + && not (onlyNumberRegex.IsMatch(content)) + +let private digitOrLetterCharRegex = + System.Text.RegularExpressions.Regex(@"^'(\d|[a-zA-Z])'$") + +let private (|CharToken|_|) token = + if token.TokenInfo.TokenName = "CHAR" + && not (digitOrLetterCharRegex.IsMatch(token.Content)) then + Some token + else + None + +let private (|StringTextToken|_|) token = + if token.TokenInfo.TokenName = "STRING_TEXT" then + Some token + else + None + +let private (|InterpStringEndToken|_|) token = + if token.TokenInfo.TokenName = "INTERP_STRING_END" then + Some token + else + None + +let escapedCharacterRegex = + System.Text.RegularExpressions.Regex("(\\\\(n|r|u|\\\"|\\\\))+") + +let rec private (|EndOfInterpolatedString|_|) tokens = + match tokens with + | StringTextToken (stToken) :: InterpStringEndToken (endToken) :: rest -> Some([ stToken ], endToken, rest) + | StringTextToken (stToken) :: EndOfInterpolatedString (stringTokens, endToken, rest) -> + Some(stToken :: stringTokens, endToken, rest) + | _ -> None + +let private (|StringText|_|) tokens = + match tokens with + | StringTextToken head :: rest -> + let stringTokens = + rest + |> List.takeWhile (fun { TokenInfo = { TokenName = tn } } -> tn = "STRING_TEXT") + |> fun others -> + let length = List.length others + let closingQuote = rest.[length] + + [ yield head + yield! others + yield closingQuote ] + + let stringContent = + let builder = StringBuilder() + + stringTokens + |> List.fold + (fun (b: StringBuilder, currentLine) st -> + if currentLine <> st.LineNumber then + let delta = st.LineNumber - currentLine + + [ 1 .. delta ] + |> List.iter (fun _ -> b.Append("\n") |> ignore) + + b.Append(st.Content), st.LineNumber + else + b.Append(st.Content), st.LineNumber) + (builder, head.LineNumber) + |> fst + |> fun b -> b.ToString() + + let stringStartIsSpecial () = + if stringContent.Length > 2 then + match stringContent.[0], stringContent.[1], stringContent.[2] with + | '@', '"', _ + | '$', '"', _ + | '"', '"', '"' -> true + | _ -> false + else + false + + let hasEscapedCharacter () = + escapedCharacterRegex.IsMatch(stringContent) + + let hasNewlines () = stringContent.Contains("\n") + let endsWithBinaryCharacter () = stringContent.EndsWith("\"B") + + if stringStartIsSpecial () + || hasEscapedCharacter () + || hasNewlines () + || endsWithBinaryCharacter () then + Some(head, stringTokens, rest, stringContent) + else + None + | _ -> None let private identIsDecompiledOperator (token: Token) = let decompiledName = @@ -643,37 +737,22 @@ let rec private getTriviaFromTokensThemSelves (allTokens: Token list) (tokens: T getTriviaFromTokensThemSelves allTokens nextRest info - | head :: rest when (head.TokenInfo.TokenName = "STRING_TEXT") -> - let stringTokens = - rest - |> List.takeWhile (fun { TokenInfo = { TokenName = tn } } -> tn = "STRING_TEXT") - |> fun others -> - let length = List.length others - let closingQuote = rest.[length] - - [ yield head - yield! others - yield closingQuote ] - + | EndOfInterpolatedString (stringTokens, interpStringEnd, rest) -> let stringContent = - let builder = StringBuilder() + [ yield! (List.map (fun t -> t.Content) stringTokens) + yield interpStringEnd.Content ] + |> String.concat String.Empty - stringTokens - |> List.fold - (fun (b: StringBuilder, currentLine) st -> - if currentLine <> st.LineNumber then - let delta = st.LineNumber - currentLine + let range = + getRangeBetween "string content" stringTokens.Head interpStringEnd - [ 1 .. delta ] - |> List.iter (fun _ -> b.Append("\n") |> ignore) + let info = + Trivia.Create(StringContent(stringContent)) range + |> List.prependItem foundTrivia - b.Append(st.Content), st.LineNumber - else - b.Append(st.Content), st.LineNumber) - (builder, head.LineNumber) - |> fst - |> fun b -> b.ToString() + getTriviaFromTokensThemSelves allTokens rest info + | StringText (head, stringTokens, rest, stringContent) -> let lastToken = List.tryLast stringTokens |> Option.defaultValue head @@ -733,7 +812,7 @@ let rec private getTriviaFromTokensThemSelves (allTokens: Token list) (tokens: T getTriviaFromTokensThemSelves allTokens rest info - | head :: rest when (head.TokenInfo.TokenName = "CHAR") -> + | CharToken (head) :: rest -> let range = getRangeBetween head.TokenInfo.TokenName head head