More refactor, overhaul, bytecode, etc

.gitignore

@@ -352,3 +352,4 @@ MigrationBackup/
 # custom hasmer ignore
 **/launchSettings.json
 /hasmer/docs
+.netcoredbg_hist

hasmer/README.md

@@ -1,6 +1,6 @@
 # hasmer
 
-This directory contains the C# source code for `hasmer`, using the .NET 5.0 platform.
+This directory contains the C# source code for `hasmer`, using the .NET 8.0 platform.
 
 # Setup
 
@@ -16,7 +16,7 @@ dotnet build
 ```
 Ensure you run the above command with the current working directory being the one containing the `hasmer.sln` file.
 
-Building with the .NET SDK CLI will generate the executable in the `.hasmer-cli/bin/Debug/net5.0` directory. You can make a release build using:
+Building with the .NET SDK CLI will generate the executable in the `.hasmer-cli/bin/Debug/net8.0` directory. You can make a release build using:
 ```
 dotnet build -c Release
 ```

hasmer/hasmer-cli/Program.cs

@@ -19,7 +19,7 @@ public class DecodeOptions {
             public bool IsApk { get; set; }
         }
 
-        [Verb("disassemble", HelpText = "Disassembles the bytecode into a Hasm file.")]
+        [Verb("disassemble", HelpText = "Disassembles Hermes bytecode into a Hasm file.")]
         public class DisassembleOptions : DecodeOptions {
             [Option("exact", Required = false, HelpText = "Disassembles the exact instructions instead of optimizing them at assemble time.")]
             public bool IsExact { get; set; }
@@ -28,7 +28,7 @@ public class DisassembleOptions : DecodeOptions {
             public bool IsVerbose { get; set; }
         }
 
-        [Verb("decompile", HelpText = "Decompiles the bytecode into a JavaScript file.")]
+        [Verb("decompile", HelpText = "Decompiles Hermes bytecode into a JavaScript file.")]
         public class DecompileOptions : DecodeOptions {
             [Option('p', "omit-protoype", Required = false, HelpText = "Omit prototypes being passed to constructors.", Default = true)]
             public bool OmitPrototypeFromConstructorInvocation { get; set; }
@@ -89,7 +89,7 @@ private static void Assemble(AssembleOptions options) {
             }
             string fileName = Path.GetFileName(options.InputPath);
             if (fileName.Contains(".")) {
-                fileName = fileName.Substring(0, fileName.IndexOf('.'));
+                fileName = fileName.Substring(0, fileName.LastIndexOf('.'));
             }
             string outputDirectory = Path.GetDirectoryName(options.InputPath);
 
@@ -110,7 +110,7 @@ private static DecoderParameters Decode(DecodeOptions options) {
 
             string fileName = Path.GetFileName(options.InputPath);
             if (fileName.Contains(".")) {
-                fileName = fileName.Substring(0, fileName.IndexOf('.'));
+                fileName = fileName.Substring(0, fileName.LastIndexOf('.'));
             }
             string outputDirectory = Path.GetDirectoryName(options.InputPath);
             byte[] hermesBytecode;
@@ -121,7 +121,8 @@ private static DecoderParameters Decode(DecodeOptions options) {
                 using MemoryStream fileStream = new MemoryStream((int)bundleEntry.UncompressedSize);
                 bundleEntry.Extract(fileStream);
                 hermesBytecode = fileStream.ToArray();
-            } else {
+            }
+            else {
                 hermesBytecode = File.ReadAllBytes(options.InputPath);
             }
 

hasmer/hasmer-cli/hasmer-cli.csproj

@@ -2,9 +2,10 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net5.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <RootNamespace>Hasmer.CLI</RootNamespace>
     <AssemblyName>hasmer</AssemblyName>
+    <PublishAot>true</PublishAot>
   </PropertyGroup>
 
   <ItemGroup>

hasmer/hasmer-lsp/hasmer-lsp.csproj

@@ -2,11 +2,12 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net5.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <RootNamespace>Hasmer.LSP</RootNamespace>
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="StreamJsonRpc" Version="2.17.8" />>
     <PackageReference Include="LspTypes" Version="3.16.6" />
   </ItemGroup>
 

hasmer/libhasmer/Assembler/DataDisassembler.cs

@@ -74,9 +74,18 @@ public PrimitiveValue[] GetElementSeries(List<HbcDataBufferItems> buffer, uint o
         private void AppendDisassembly(StringBuilder builder, List<HbcDataBufferItems> buffer, char prefix) {
             for (int i = 0; i < buffer.Count; i++) {
                 HbcDataBufferItems items = buffer[i];
+                switch (items.Prefix.TagType) {
+                    case HbcDataBufferTagType.Null:
+                    case HbcDataBufferTagType.True:
+                    case HbcDataBufferTagType.False:
+                        builder.AppendLine($".data {prefix}{i} {items.Prefix.TagType}[{items.Items.Length}]");
+                        continue;
+                    default:
+                        break;
+                }
                 IEnumerable<PrimitiveValue> mapped = items.Items.Select(x => {
                     if (x.TypeCode == TypeCode.String) {
-                        x.SetValue('"' + x.GetValue<string>().Replace("\"", "\\\"") + '"');
+                        x.SetValue(StringEscape.Escape(x.GetValue<string>()));
                     }
                     return x;
                 });
@@ -85,16 +94,19 @@ private void AppendDisassembly(StringBuilder builder, List<HbcDataBufferItems> b
                     _ => items.Prefix.TagType.ToString()
                 };
                 string joined = string.Join(", ", mapped);
-                builder.AppendLine($".data {prefix}{i} {tagType}[{joined}] # offset = {items.Offset}");
+                builder.AppendLine($".data {prefix}{i} {tagType}[] {{ {joined} }}");
             }
             if (buffer.Count > 0) {
                 builder.AppendLine();
             }
         }
 
         public void DisassembleData() {
+            Console.WriteLine("Parsing array buffer...");
             ArrayBuffer = Source.ArrayBuffer.ReadAll(Source);
+            Console.WriteLine("Parsing object key buffer...");
             KeyBuffer = Source.ObjectKeyBuffer.ReadAll(Source);
+            Console.WriteLine("Parsing object value buffer...");
             ValueBuffer = Source.ObjectValueBuffer.ReadAll(Source);
         }
 

hasmer/libhasmer/Assembler/FunctionDisassembler.cs

@@ -178,7 +178,7 @@ private string AnnotateObject(uint keyBufferIndex, uint valueBufferIndex, ushort
         /// Adds a comment to an instruction if neccessary.
         /// </summary>
         private void AnnotateVerbose(SourceCodeBuilder builder, HbcInstruction insn) {
-            builder.Write("# offset = 0x");
+            builder.Write("// offset = 0x");
             builder.Write(insn.Offset.ToString("X"));
             builder.Write(", length = ");
             builder.Write(insn.Length.ToString());
@@ -205,7 +205,7 @@ private void AnnotateInstruction(SourceCodeBuilder builder, HbcInstruction insn)
             };
 
             if (annotation != null) {
-                builder.Write("# ");
+                builder.Write("// ");
                 builder.Write(annotation);
             }
         }

hasmer/libhasmer/Assembler/HbcAssembler.cs

@@ -12,16 +12,6 @@ namespace Hasmer.Assembler {
     /// Represents a Hasm assembler, which assembles into Hermes bytecode.
     /// </summary>
     public class HbcAssembler {
-        /// <summary>
-        /// Represents the data contained with in the header of the Hasm file (i.e. the initial ".hasm" declaration)./
-        /// </summary>
-        public HasmHeaderReader Header { get; set; }
-
-        /// <summary>
-        /// The file builder instance which creates the final <see cref="HbcFile"/> to be serialized.
-        /// </summary>
-        public HbcFileBuilder FileBuilder { get; set; }
-
         /// <summary>
         /// The data assembler instance.
         /// </summary>
@@ -32,6 +22,13 @@ public class HbcAssembler {
         /// </summary>
         public FunctionAssembler FunctionAssembler { get; set; }
 
+        /// <summary>
+        /// The Hermes bytecode file being assembled.
+        /// </summary>
+        public HbcFile File { get; set; }
+
+        public bool IsExact { get; set; }
+
         /// <summary>
         /// The Hasm assembly to disassemble.
         /// </summary>
@@ -48,21 +45,41 @@ public HbcAssembler(string source) {
         /// Assembles the given Hasm assembly into a Hermes bytecode file, serialized into a byte array.
         /// </summary>
         public byte[] Assemble() {
-            HasmTokenStream stream = new HasmTokenStream(Source);
+            Console.WriteLine("Parsing Hasm file...");
+            HasmTokenizer tokenizer = new HasmTokenizer();
+            tokenizer.TokenizeProgram(Source);
+
+            // List<HasmToken> tokens = null;
+
+            // IsExact = tokenStream.State.IsExact;
+            // File = new HbcFile {
+            //     Header = new HbcHeader {
+            //         Magic = HbcHeader.HBC_MAGIC_HEADER,
+            //         Version = tokenStream.State.BytecodeFormat.Version,
+            //         SourceHash = new byte[20],
+            //         GlobalCodeIndex = 0,
+            //         Padding = new byte[31],
+            //     },
+            //     BytecodeFormat = tokenStream.State.BytecodeFormat,
+            // };
 
-            Header = new HasmHeaderReader(stream);
-            Header.Read();
+            // Console.WriteLine("Assembling data...");
+            // DataAssembler = new DataAssembler(tokens);
+            // DataAssembler.Assemble();
 
-            FileBuilder = new HbcFileBuilder(this);
+            // Console.WriteLine("Assembling functions...");
+            // FunctionAssembler = new FunctionAssembler(this, tokens);
+            // FunctionAssembler.Assemble();
 
-            DataAssembler = new DataAssembler(stream);
-            DataAssembler.Assemble();
+            // Console.WriteLine("Assembling Hermes bytecode file...");
+            // File.StringTable = DataAssembler.StringTable.ToArray();
+            // File.ArrayBuffer = new HbcDataBuffer(DataAssembler.ArrayBuffer.RawBuffer.ToArray());
+            // File.ObjectKeyBuffer = new HbcDataBuffer(DataAssembler.ObjectKeyBuffer.RawBuffer.ToArray());
+            // File.ObjectValueBuffer = new HbcDataBuffer(DataAssembler.ObjectValueBuffer.RawBuffer.ToArray());
 
-            FunctionAssembler = new FunctionAssembler(this, stream);
-            FunctionAssembler.Assemble();
+            // return File.Write();
 
-            HbcFile hbcFile = FileBuilder.Build();
-            return hbcFile.Write();
+            return new byte[0];
         }
     }
 }

hasmer/libhasmer/Assembler/HbcDisassembler.cs

@@ -50,11 +50,18 @@ public string Disassemble() {
             builder.AppendLine();
             builder.AppendLine(DataDisassembler.Disassemble());
 
-            foreach (HbcSmallFuncHeader func in Source.SmallFuncHeaders) {
-                FunctionDisassembler decompiler = new FunctionDisassembler(this, func.GetAssemblerHeader());
-                builder.AppendLine(decompiler.Disassemble());
-                builder.AppendLine();
+            Console.Write("Disassembling functions... ");
+            using (ConsoleProgressBar progress = new ConsoleProgressBar()) {
+                for (int i = 0; i < Source.SmallFuncHeaders.Length; i++) {
+                    progress.Report(i / (double)Source.SmallFuncHeaders.Length);
+
+                    HbcSmallFuncHeader func = Source.SmallFuncHeaders[i];
+                    FunctionDisassembler decompiler = new FunctionDisassembler(this, func.GetAssemblerHeader());
+                    builder.AppendLine(decompiler.Disassemble());
+                    builder.AppendLine();
+                }
             }
+            Console.WriteLine("done!");
 
             return builder.ToString();
         }

hasmer/libhasmer/Assembler/Parser/HasmCommentParser.cs

@@ -6,20 +6,29 @@
 
 namespace Hasmer.Assembler.Parser {
     /// <summary>
-    /// Parses a comment (starting with a '#') from the stream.
+    /// Parses a comment (starting with a '//') from the stream.
     /// </summary>
     public class HasmCommentParser : IHasmTokenParser {
         public bool CanParse(HasmReaderState asm) {
-            return asm.Stream.PeekCharacters(1) == "#";
+            return asm.Stream.Peek(2) == "//";
         }
 
         public HasmToken Parse(HasmReaderState asm) {
             if (!CanParse(asm)) {
                 throw new HasmParserException(asm.Stream, "invalid comment");
             }
 
-            asm.Stream.CurrentLine++;
-            asm.Stream.CurrentColumn = 0;
+            while (true) {
+                char c = asm.Stream.PeekChar();
+                if (c == '\0') {
+                    break;
+                }
+                asm.Stream.Advance(1);
+                if (c == '\n') {
+                    break;
+                }
+            }
+
             if (!asm.Stream.IsFinished) {
                 asm.Stream.SkipWhitespace();
             }

hasmer/libhasmer/Assembler/Parser/HasmDataDeclaration.cs

@@ -0,0 +1,43 @@
+using System.Collections.Generic;
+
+namespace Hasmer.Assembler.Parser {
+    /// <summary>
+    /// The kind of data being declared in a ".data" declaration.
+    /// </summary>
+    [Newtonsoft.Json.JsonConverter(typeof(Newtonsoft.Json.Converters.StringEnumConverter))]
+    public enum HasmDataDeclarationKind {
+        String,
+        Integer,
+        Number,
+        Null,
+        True,
+        False
+    }
+
+    /// <summary>
+    /// Represents a ".data" declaration.
+    /// </summary>
+    public class HasmDataDeclaration {
+        /// <summary>
+        /// The label of the data.
+        /// </summary>
+        public HasmLabelToken Label { get; set; }
+
+        /// <summary>
+        /// The kind of the data.
+        /// </summary>
+        public HasmDataDeclarationKind Kind { get; set; }
+
+        /// <summary>
+        /// The amount of distinct items in the data declaration.
+        /// In declarations where there are specific elements, this is equal to <see cref="Elements.Length" />.
+        /// In declarations where elements are repeated (i.e. Null, True, or False), this is equal to the supplied repeat count.
+        /// </summary>
+        public int Count { get; set; }
+
+        /// <summary>
+        /// The tokens that define the contents of the data, if there are specific elements.
+        /// </summary>
+        public List<HasmLiteralToken> Elements { get; set; }
+    }
+}