Feature/generated lexer (#77)

* #57 - абстракция IStructure.cs

* #57 - token types provider

* #57 - DI refactoring

* #57 - json highlight

* #57 - абстракция IGeneratedRegexContainer

* fix

* #57 - SG project

* #57 - fix SG launch

* #57 - tests

* #57 - удаление динамического построения RegEx

* #57 - убрал лишнее из TokenType

* #57 - тест на проверку подстановки

* fix

.github/workflows/develop.yml

@@ -37,6 +37,7 @@ jobs:
         run: dotnet build --no-restore -c Release -v n
       - name: Unit Tests
         run: |
+          dotnet test -c Release ./tests/HydraScript.Infrastructure.LexerRegexGenerator.Tests/HydraScript.Infrastructure.LexerRegexGenerator.Tests.csproj --no-build -v n
           dotnet test -c Release /p:CollectCoverage=true /p:CoverletOutputFormat=cobertura --no-build -v n --filter="Category=Unit"
           mkdir coverage-report
       - name: Code Coverage Summary Report For Merge Request

ExtendedJavaScriptSubset.sln

@@ -88,6 +88,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HydraScript.Infrastructure"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HydraScript.IntegrationTests", "tests\HydraScript.IntegrationTests\HydraScript.IntegrationTests.csproj", "{1CE98127-3027-4BD4-AAA3-63A589B09E73}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HydraScript.Infrastructure.LexerRegexGenerator", "src\Infrastructure\HydraScript.Infrastructure.LexerRegexGenerator\HydraScript.Infrastructure.LexerRegexGenerator.csproj", "{74D1495B-12A4-4E1A-ABE0-93029ECDC5FE}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HydraScript.Infrastructure.LexerRegexGenerator.Tests", "tests\HydraScript.Infrastructure.LexerRegexGenerator.Tests\HydraScript.Infrastructure.LexerRegexGenerator.Tests.csproj", "{829111AD-4A5C-4B3D-AC28-208309CE10D6}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -130,6 +134,14 @@ Global
 		{1CE98127-3027-4BD4-AAA3-63A589B09E73}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{1CE98127-3027-4BD4-AAA3-63A589B09E73}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{1CE98127-3027-4BD4-AAA3-63A589B09E73}.Release|Any CPU.Build.0 = Release|Any CPU
+		{74D1495B-12A4-4E1A-ABE0-93029ECDC5FE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{74D1495B-12A4-4E1A-ABE0-93029ECDC5FE}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{74D1495B-12A4-4E1A-ABE0-93029ECDC5FE}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{74D1495B-12A4-4E1A-ABE0-93029ECDC5FE}.Release|Any CPU.Build.0 = Release|Any CPU
+		{829111AD-4A5C-4B3D-AC28-208309CE10D6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{829111AD-4A5C-4B3D-AC28-208309CE10D6}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{829111AD-4A5C-4B3D-AC28-208309CE10D6}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{829111AD-4A5C-4B3D-AC28-208309CE10D6}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(NestedProjects) = preSolution
 		{54CBE5A7-3C3E-44ED-B877-7B08A818083B} = {86CF2A2F-4DFE-48E7-B062-EF824730916A}
@@ -147,5 +159,7 @@ Global
 		{07DEA726-9772-4B61-AA66-1B82823F6BF0} = {A280C99E-0112-4F3C-A5E9-B793F2D9898E}
 		{160B77ED-BA30-40A2-81EF-F5D1E4D22039} = {B7DDF6C9-B67C-430A-948A-A380EF68DEF1}
 		{1CE98127-3027-4BD4-AAA3-63A589B09E73} = {3F131901-A9EC-451A-B7E9-726887CFE5FB}
+		{74D1495B-12A4-4E1A-ABE0-93029ECDC5FE} = {B7DDF6C9-B67C-430A-948A-A380EF68DEF1}
+		{829111AD-4A5C-4B3D-AC28-208309CE10D6} = {3F131901-A9EC-451A-B7E9-726887CFE5FB}
 	EndGlobalSection
 EndGlobal

src/Application/HydraScript.Application.CodeGeneration/HydraScript.Application.CodeGeneration.csproj

@@ -4,7 +4,7 @@
         <TargetFramework>net8.0</TargetFramework>
         <ImplicitUsings>enable</ImplicitUsings>
         <Nullable>enable</Nullable>
-        <TreatWarningsAsErrors>True</TreatWarningsAsErrors>
+        <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
     </PropertyGroup>
 
     <ItemGroup>

src/Application/HydraScript.Application.StaticAnalysis/HydraScript.Application.StaticAnalysis.csproj

@@ -4,7 +4,7 @@
         <TargetFramework>net8.0</TargetFramework>
         <ImplicitUsings>enable</ImplicitUsings>
         <Nullable>enable</Nullable>
-        <TreatWarningsAsErrors>True</TreatWarningsAsErrors>
+        <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
     </PropertyGroup>
 
     <ItemGroup>

src/Domain/HydraScript.Domain.BackEnd/HydraScript.Domain.BackEnd.csproj

@@ -4,7 +4,7 @@
         <TargetFramework>net8.0</TargetFramework>
         <ImplicitUsings>enable</ImplicitUsings>
         <Nullable>enable</Nullable>
-        <TreatWarningsAsErrors>True</TreatWarningsAsErrors>
+        <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
     </PropertyGroup>
 
 </Project>

src/Domain/HydraScript.Domain.FrontEnd/HydraScript.Domain.FrontEnd.csproj

@@ -4,7 +4,7 @@
         <TargetFramework>net8.0</TargetFramework>
         <ImplicitUsings>enable</ImplicitUsings>
         <Nullable>enable</Nullable>
-        <TreatWarningsAsErrors>True</TreatWarningsAsErrors>
+        <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
     </PropertyGroup>
 
     <ItemGroup>

src/Domain/HydraScript.Domain.FrontEnd/Lexer/IGeneratedRegexContainer.cs

@@ -0,0 +1,8 @@
+using System.Text.RegularExpressions;
+
+namespace HydraScript.Domain.FrontEnd.Lexer;
+
+public interface IGeneratedRegexContainer
+{
+    public static abstract Regex GetRegex();
+}

src/Domain/HydraScript.Domain.FrontEnd/Lexer/ILexer.cs

@@ -2,7 +2,7 @@ namespace HydraScript.Domain.FrontEnd.Lexer;
 
 public interface ILexer
 {
-    public Structure Structure { get; }
+    public IStructure Structure { get; }
 
     public List<Token> GetTokens(string text);
 }

src/Domain/HydraScript.Domain.FrontEnd/Lexer/IStructure.cs

@@ -0,0 +1,11 @@
+using System.Text.RegularExpressions;
+using HydraScript.Domain.FrontEnd.Lexer.TokenTypes;
+
+namespace HydraScript.Domain.FrontEnd.Lexer;
+
+public interface IStructure : IEnumerable<TokenType>
+{
+    public Regex Regex { get; }
+
+    public TokenType FindByTag(string tag);
+}

src/Domain/HydraScript.Domain.FrontEnd/Lexer/ITokenTypesProvider.cs

@@ -0,0 +1,8 @@
+using HydraScript.Domain.FrontEnd.Lexer.TokenTypes;
+
+namespace HydraScript.Domain.FrontEnd.Lexer;
+
+public interface ITokenTypesProvider
+{
+    IEnumerable<TokenType> GetTokenTypes();
+}

src/Domain/HydraScript.Domain.FrontEnd/Lexer/Impl/RegexLexer.cs

@@ -4,12 +4,12 @@
 
 namespace HydraScript.Domain.FrontEnd.Lexer.Impl;
 
-public class RegexLexer(Structure structure, ITextCoordinateSystemComputer computer) : ILexer, IEnumerable<Token>
+public class RegexLexer(IStructure structure, ITextCoordinateSystemComputer computer) : ILexer, IEnumerable<Token>
 {
     private IReadOnlyList<int> _lines = [];
     private string _text = "";
 
-    public Structure Structure { get; } = structure;
+    public IStructure Structure { get; } = structure;
 
     public List<Token> GetTokens(string text)
     {

src/Domain/HydraScript.Domain.FrontEnd/Lexer/Impl/Structure.cs

@@ -0,0 +1,31 @@
+using System.Collections;
+using System.Diagnostics.CodeAnalysis;
+using System.Text;
+using System.Text.RegularExpressions;
+using HydraScript.Domain.FrontEnd.Lexer.TokenTypes;
+
+namespace HydraScript.Domain.FrontEnd.Lexer.Impl;
+
+public class Structure<TContainer>(ITokenTypesProvider provider) : IStructure
+    where TContainer : IGeneratedRegexContainer
+{
+    private Dictionary<string, TokenType> Types { get; } = provider.GetTokenTypes()
+        .Concat([new EndOfProgramType(), new ErrorType()])
+        .ToDictionary(x => x.Tag);
+
+    public Regex Regex { get; } = TContainer.GetRegex();
+
+    public TokenType FindByTag(string tag) =>
+        Types[tag];
+
+    public override string ToString() =>
+        new StringBuilder()
+            .AppendJoin('\n', this)
+            .ToString();
+
+    public IEnumerator<TokenType> GetEnumerator() =>
+        Types.Values.GetEnumerator();
+
+    [ExcludeFromCodeCoverage]
+    IEnumerator IEnumerable.GetEnumerator() => GetEnumerator();
+}

src/Domain/HydraScript.Domain.FrontEnd/Lexer/TokenTypes/EndOfProgramType.cs

@@ -1,6 +1,6 @@
 namespace HydraScript.Domain.FrontEnd.Lexer.TokenTypes;
 
-internal record EndOfProgramType() : TokenType("EOP", "", int.MaxValue - 1)
+internal record EndOfProgramType() : TokenType(EopTag)
 {
-    public override bool EndOfProgram() => true;
+    public const string EopTag = "EOP";
 }

src/Domain/HydraScript.Domain.FrontEnd/Lexer/TokenTypes/ErrorType.cs

@@ -1,6 +1,6 @@
 namespace HydraScript.Domain.FrontEnd.Lexer.TokenTypes;
 
-internal record ErrorType() : TokenType("ERROR", @"\S+", int.MaxValue)
+internal record ErrorType() : TokenType("ERROR")
 {
     public override bool Error() => true;
 }

src/Domain/HydraScript.Domain.FrontEnd/Lexer/TokenTypes/IgnorableType.cs

@@ -1,7 +1,6 @@
 namespace HydraScript.Domain.FrontEnd.Lexer.TokenTypes;
 
-public record IgnorableType(string Tag, string Pattern, int Priority)
-    : TokenType(Tag, Pattern, Priority)
+public record IgnorableType(string Tag) : TokenType(Tag)
 {
     public override bool CanIgnore() => true;
 }

src/Domain/HydraScript.Domain.FrontEnd/Lexer/TokenTypes/TokenType.cs

@@ -1,14 +1,10 @@
 namespace HydraScript.Domain.FrontEnd.Lexer.TokenTypes;
 
-public record TokenType(string Tag, string Pattern, int Priority)
+public record TokenType(string Tag)
 {
     public virtual bool CanIgnore() => false;
 
-    public virtual bool EndOfProgram() => false;
-
     public virtual bool Error() => false;
 
-    public string GetNamedRegex() => $"(?<{Tag}>{Pattern})";
-
-    public override string ToString() => Tag;
+    public sealed override string ToString() => Tag;
 }

src/Domain/HydraScript.Domain.FrontEnd/Lexer/TokenTypesJson.cs

@@ -1,7 +1,10 @@
+using System.Diagnostics.CodeAnalysis;
+
 namespace HydraScript.Domain.FrontEnd.Lexer;
 
 public static class TokenTypesJson
 {
+    [StringSyntax(StringSyntaxAttribute.Json)]
     public const string String =
         """
         [

src/Domain/HydraScript.Domain.FrontEnd/Parser/Impl/TopDownParser.cs

@@ -1,6 +1,7 @@
 using System.Globalization;
 using System.Text.RegularExpressions;
 using HydraScript.Domain.FrontEnd.Lexer;
+using HydraScript.Domain.FrontEnd.Lexer.TokenTypes;
 using HydraScript.Domain.FrontEnd.Parser.Impl.Ast;
 using HydraScript.Domain.FrontEnd.Parser.Impl.Ast.Nodes;
 using HydraScript.Domain.FrontEnd.Parser.Impl.Ast.Nodes.Declarations;
@@ -26,7 +27,7 @@ public IAbstractSyntaxTree Parse(string text)
         _tokens = _lexer.GetTokens(text);
 
         var root = Script();
-        Expect("EOP");
+        Expect(EndOfProgramType.EopTag);
         return new AbstractSyntaxTree(root);
     }
 

src/Domain/HydraScript.Domain.IR/HydraScript.Domain.IR.csproj

@@ -4,7 +4,7 @@
         <TargetFramework>net8.0</TargetFramework>
         <ImplicitUsings>enable</ImplicitUsings>
         <Nullable>enable</Nullable>
-        <TreatWarningsAsErrors>True</TreatWarningsAsErrors>
+        <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
     </PropertyGroup>
 
 </Project>

src/HydraScript/HydraScript.csproj

@@ -6,7 +6,7 @@
         <Version>1.2.6</Version>
         <ImplicitUsings>enable</ImplicitUsings>
         <Nullable>enable</Nullable>
-        <TreatWarningsAsErrors>True</TreatWarningsAsErrors>
+        <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
     </PropertyGroup>
 
     <ItemGroup>

src/Infrastructure/HydraScript.Infrastructure.LexerRegexGenerator/HydraScript.Infrastructure.LexerRegexGenerator.csproj

@@ -0,0 +1,25 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <TargetFramework>net8.0</TargetFramework>
+        <Nullable>enable</Nullable>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
+
+        <EnforceExtendedAnalyzerRules>true</EnforceExtendedAnalyzerRules>
+        <IsRoslynComponent>true</IsRoslynComponent>
+    </PropertyGroup>
+
+    <ItemGroup>
+        <PackageReference Include="Microsoft.CodeAnalysis.Analyzers" Version="3.3.4">
+            <PrivateAssets>all</PrivateAssets>
+            <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+        </PackageReference>
+        <PackageReference Include="Microsoft.CodeAnalysis.CSharp" Version="4.10.0"/>
+        <PackageReference Include="Microsoft.CodeAnalysis.CSharp.Workspaces" Version="4.10.0"/>
+        <PackageReference Include="System.Text.Json" Version="8.0.4">
+            <PrivateAssets>all</PrivateAssets>
+        </PackageReference>
+    </ItemGroup>
+
+</Project>

src/Infrastructure/HydraScript.Infrastructure.LexerRegexGenerator/PatternGenerator.TokenTypes.cs

@@ -0,0 +1,49 @@
+using System.Diagnostics.CodeAnalysis;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+namespace HydraScript.Infrastructure.LexerRegexGenerator;
+
+public partial class PatternGenerator
+{
+    private static readonly JsonSerializerOptions JsonSerializerOptions = new()
+    {
+        Converters = { new TokenTypesReadConverter() }
+    };
+
+    private record TokenType(
+        string Tag,
+        string Pattern,
+        int Priority)
+    {
+        public string GetNamedRegex() => $"(?<{Tag}>{Pattern})";
+    }
+
+    [ExcludeFromCodeCoverage]
+    private class TokenTypesReadConverter : JsonConverter<IEnumerable<TokenType>>
+    {
+        public override IEnumerable<TokenType> Read(
+            ref Utf8JsonReader reader,
+            Type typeToConvert,
+            JsonSerializerOptions options)
+        {
+            var root = JsonElement.ParseValue(ref reader);
+            var tokenTypes = root.EnumerateArray()
+                .Select(element =>
+                {
+                    var tag = element.GetProperty("tag").GetString()!;
+                    var pattern = element.GetProperty("pattern").GetString()!;
+                    var priority = element.GetProperty("priority").GetInt32();
+
+                    return new TokenType(tag, pattern, priority);
+                })
+                .OrderBy(x => x.Priority);
+            return tokenTypes;
+        }
+
+        public override void Write(
+            Utf8JsonWriter writer,
+            IEnumerable<TokenType> value,
+            JsonSerializerOptions options) => throw new NotSupportedException();
+    }
+}