Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Final lexer changes #11078

Merged
merged 4 commits into from
Oct 24, 2024
Merged

Final lexer changes #11078

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
2033814
Switch the new lexer off-by-default.
333fred Oct 23, 2024
078effe
Add a new document for the lexer breaking changes around pragmas.
333fred Oct 23, 2024
0419a03
Update comment.
333fred Oct 23, 2024
19e376f
Feedback
333fred Oct 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
Switch the new lexer off-by-default.
  • Loading branch information
@333fred
333fred committed Oct 23, 2024
commit 20338145fe260529491cf5d2ae9b33623f14a259
10 changes: 5 additions & 5 deletions ...t.CodeAnalysis.Razor.Compiler/src/SourceGenerators/RazorSourceGenerator.RazorProviders.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,15 +49,15 @@ public partial class RazorSourceGenerator
.Where(r => r.Display is { } display && display.EndsWith("Microsoft.AspNetCore.Components.dll", StringComparison.Ordinal))
.ToImmutableArray();

var isComponentParameterSupported = minimalReferences.Length == 0
? false
var isComponentParameterSupported = minimalReferences.Length == 0
? false
: CSharpCompilation.Create("components", references: minimalReferences).HasAddComponentParameter();

var razorConfiguration = new RazorConfiguration(razorLanguageVersion, configurationName ?? "default", Extensions: [], UseConsolidatedMvcViews: true, SuppressAddComponentParameter: !isComponentParameterSupported);

// We use the new tokenizer by default
var useRazorTokenizer = !parseOptions.Features.TryGetValue("use-razor-tokenizer", out var useRazorTokenizerValue)
|| !string.Equals(useRazorTokenizerValue, "false", StringComparison.OrdinalIgnoreCase);
var useRoslynTokenizer = parseOptions.Features.TryGetValue("use-roslyn-tokenizer", out var useRazorTokenizerValue)
&& string.Equals(useRazorTokenizerValue, "true", StringComparison.OrdinalIgnoreCase);

var razorSourceGenerationOptions = new RazorSourceGenerationOptions()
{
Expand All @@ -67,7 +67,7 @@ public partial class RazorSourceGenerator
SupportLocalizedComponentNames = supportLocalizedComponentNames == "true",
CSharpParseOptions = (CSharpParseOptions)parseOptions,
TestSuppressUniqueIds = _testSuppressUniqueIds,
UseRoslynTokenizer = useRazorTokenizer,
UseRoslynTokenizer = useRoslynTokenizer,
};

return (razorSourceGenerationOptions, diagnostic);
Expand Down
154 changes: 154 additions & 0 deletions ...Compiler/test/Microsoft.NET.Sdk.Razor.SourceGenerators.Tests/RazorSourceGeneratorTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3265,5 +3265,159 @@ public async Task IncrementalCompilation_OnlyCompilationRuns_When_MetadataRefere
e => Assert.Equal("DiscoverTagHelpersFromCompilationStart", e.EventName),
e => Assert.Equal("DiscoverTagHelpersFromCompilationStop", e.EventName));
}

[Theory]
[InlineData("true")]
[InlineData("True")]
[InlineData("TRUE")]
[InlineData("tRuE")]
public async Task RoslynTokenizerEnabledWithTrue(string value)
{
var parseOptions = CSharpParseOptions.Default.WithFeatures([new("use-roslyn-tokenizer", value)]);

var project = CreateTestProject(new()
{
["Pages/Index.razor"] = """"
<div>@("""
nested "
""")</div>
"""",
}, cSharpParseOptions: parseOptions);

var compilation = await project.GetCompilationAsync();
var (driver, additionalTexts) = await GetDriverWithAdditionalTextAsync(project);

var result = RunGenerator(compilation!, ref driver);
result.VerifyPageOutput(
""""
#pragma checksum "Pages/Index.razor" "{ff1816ec-aa5e-4d10-87f7-6f4963833460}" "c6855f3cabbcb69477e3f5a61f8d77fcfed086c2"
// <auto-generated/>
#pragma warning disable 1591
namespace MyApp.Pages
{
#line default
using global::System;
using global::System.Collections.Generic;
using global::System.Linq;
using global::System.Threading.Tasks;
using global::Microsoft.AspNetCore.Components;
#line default
#line hidden
#nullable restore
public partial class Index : global::Microsoft.AspNetCore.Components.ComponentBase
#nullable disable
{
#pragma warning disable 1998
protected override void BuildRenderTree(global::Microsoft.AspNetCore.Components.Rendering.RenderTreeBuilder __builder)
{
__builder.OpenElement(0, "div");
__builder.AddContent(1,
#nullable restore
#line (1,8)-(3,8) "Pages/Index.razor"
"""
nested "
"""

#line default
#line hidden
#nullable disable
);
__builder.CloseElement();
}
#pragma warning restore 1998
}
}
#pragma warning restore 1591

"""");
}

[Theory]
[InlineData("false")]
[InlineData("False")]
[InlineData("FALSE")]
[InlineData("FaLsE")]
[InlineData("")]
[InlineData(null)]
public async Task RoslynTokenizerDisabledWithFalseOrNothing(string? value)
{
var parseOptions = CSharpParseOptions.Default;

if (value != null)
{
parseOptions = parseOptions.WithFeatures([new("use-roslyn-tokenizer", value)]);
}

var project = CreateTestProject(new()
{
["Pages/Index.razor"] = """"
<div>@("""
nested "
""")</div>
"""",
}, cSharpParseOptions: parseOptions);

var compilation = await project.GetCompilationAsync();
var (driver, additionalTexts) = await GetDriverWithAdditionalTextAsync(project);

var result = RunGenerator(compilation!, ref driver,
// Pages/Index.razor(3,10): error CS1525: Invalid expression term '/'
// """)</div>
Diagnostic(ErrorCode.ERR_InvalidExprTerm, "/").WithArguments("/").WithLocation(3, 10),
// Pages/Index.razor(3,11): error CS0103: The name 'div' does not exist in the current context
// """)</div>
Diagnostic(ErrorCode.ERR_NameNotInContext, "div").WithArguments("div").WithLocation(3, 11),
// Pages/Index.razor(3,15): error CS1525: Invalid expression term ')'
// """)</div>
Diagnostic(ErrorCode.ERR_InvalidExprTerm, "").WithArguments(")").WithLocation(3, 15),
// Pages/Index.razor(3,15): error CS1002: ; expected
// """)</div>
Diagnostic(ErrorCode.ERR_SemicolonExpected, "").WithLocation(3, 15),
// Pages/Index.razor(3,15): error CS1513: } expected
// """)</div>
Diagnostic(ErrorCode.ERR_RbraceExpected, "").WithLocation(3, 15));

result.VerifyPageOutput(
""""
#pragma checksum "Pages/Index.razor" "{ff1816ec-aa5e-4d10-87f7-6f4963833460}" "c6855f3cabbcb69477e3f5a61f8d77fcfed086c2"
// <auto-generated/>
#pragma warning disable 1591
namespace MyApp.Pages
{
#line default
using global::System;
using global::System.Collections.Generic;
using global::System.Linq;
using global::System.Threading.Tasks;
using global::Microsoft.AspNetCore.Components;
#line default
#line hidden
#nullable restore
public partial class Index : global::Microsoft.AspNetCore.Components.ComponentBase
#nullable disable
{
#pragma warning disable 1998
protected override void BuildRenderTree(global::Microsoft.AspNetCore.Components.Rendering.RenderTreeBuilder __builder)
{
__builder.OpenElement(0, "div");
__builder.AddContent(1,
#nullable restore
#line (1,8)-(3,15) "Pages/Index.razor"
"""
nested "
""")</div>
#line default
#line hidden
#nullable disable
);
__builder.CloseElement();
}
#pragma warning restore 1998
}
}
#pragma warning restore 1591

"""");
}
}
}
9 changes: 5 additions & 4 deletions ...iler/test/Microsoft.NET.Sdk.Razor.SourceGenerators.Tests/RazorSourceGeneratorTestsBase.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -230,9 +230,10 @@ protected static async Task<string> VerifyRazorPageMatchesBaselineAsync(Compilat

protected static Project CreateTestProject(
OrderedStringDictionary additionalSources,
OrderedStringDictionary? sources = null)
OrderedStringDictionary? sources = null,
CSharpParseOptions? cSharpParseOptions = null)
{
var project = CreateBaseProject();
var project = CreateBaseProject(cSharpParseOptions);

if (sources is not null)
{
Expand Down Expand Up @@ -294,7 +295,7 @@ public bool TryResolveAssemblyPaths(CompilationLibrary library, List<string>? as
}
}

private static Project CreateBaseProject()
private static Project CreateBaseProject(CSharpParseOptions? cSharpParseOptions)
{
var projectId = ProjectId.CreateNewId(debugName: "TestProject");

Expand All @@ -314,7 +315,7 @@ private static Project CreateBaseProject()
new("CS8019", ReportDiagnostic.Suppress),
}));

project = project.WithParseOptions(((CSharpParseOptions)project.ParseOptions!).WithLanguageVersion(LanguageVersion.Preview));
project = project.WithParseOptions(cSharpParseOptions ?? ((CSharpParseOptions)project.ParseOptions!).WithLanguageVersion(LanguageVersion.Preview));

foreach (var defaultCompileLibrary in DependencyContext.Load(typeof(RazorSourceGeneratorTests).Assembly)!.CompileLibraries)
{
Expand Down