Upgrade parser to ts and running file produces file.tokens.json and file.ast_tree.json

Author: Zombi

.gitignore

@@ -1,2 +0,0 @@
-bin
-obj

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+    "deno.enable": true
+}

Soup.cs

@@ -0,0 +1,19 @@
+import { Tokenize } from "./lexer/lexer.ts";
+import Parser from "./parser/parser.ts";
+
+const filecontents = Deno.readTextFileSync(Deno.args[0]);
+const src = filecontents.split("");
+const parser = new Parser();
+const Tokens = Tokenize(src)
+
+for (let index = 0; index < Tokens.length; index++) {
+    const item = Tokens[index];
+    console.log("Value: '"+item.T_Value + "' Token: " + item.T_Type);
+    
+}
+
+const program = parser.produceAST(src)
+console.log(program)
+
+Deno.writeTextFileSync(Deno.args[0].replaceAll(".sp", ".tokens.json"), JSON.stringify(Tokens, null, 3));
+Deno.writeTextFileSync(Deno.args[0].replaceAll(".sp", ".ast_tree.json"), JSON.stringify(program, null, 3));

Soup.csproj

@@ -0,0 +1,5 @@
+{
+  "tasks": {
+    "dev": "deno run --watch main.ts"
+  }
+}

Soup.ts

@@ -0,0 +1,81 @@
+import { TokenTypes, Token, BuildToken, IsKeyword, IsAlphabetical, IsNumerical, IsOneCharToken, IsSkippable } from "./tokens.ts";
+
+export function Tokenize(chars: string[]){
+
+    let EOF = 0;
+    let tokens = new Array<Token>;
+    let ip = 0;
+    while (ip <= chars.length-1){
+
+        if (EOF){
+            break;
+        }
+
+        if (IsOneCharToken(chars[ip])){
+
+            if (chars[ip] == "("){tokens=BuildToken(tokens, chars[ip], TokenTypes.LParen);if (ip+1>(chars.length-1)){EOF=1; break;}else{ip++;}}
+            if (chars[ip] == ")"){tokens=BuildToken(tokens, chars[ip], TokenTypes.RParen);if (ip+1>(chars.length-1)){EOF=1; break;}else{ip++;}}
+            if (chars[ip] == "{"){tokens=BuildToken(tokens, chars[ip], TokenTypes.LBrace);if (ip+1>(chars.length-1)){EOF=1; break;}else{ip++;}}
+            if (chars[ip] == "}"){tokens=BuildToken(tokens, chars[ip], TokenTypes.RBrace);if (ip+1>(chars.length-1)){EOF=1; break;}else{ip++;}}
+            if (chars[ip] == "<"){tokens=BuildToken(tokens, chars[ip], TokenTypes.LArrow);if (ip+1>(chars.length-1)){EOF=1; break;}else{ip++;}}
+            if (chars[ip] == ">"){tokens=BuildToken(tokens, chars[ip], TokenTypes.RArrow);if (ip+1>(chars.length-1)){EOF=1; break;}else{ip++;}}
+            if (chars[ip] == "["){tokens=BuildToken(tokens, chars[ip], TokenTypes.LBracket);if (ip+1>(chars.length-1)){EOF=1; break;}else{ip++;}}
+            if (chars[ip] == "]"){tokens=BuildToken(tokens, chars[ip], TokenTypes.RBracket);if (ip+1>(chars.length-1)){EOF=1; break;}else{ip++;}}
+            if (chars[ip] == "="){tokens=BuildToken(tokens, chars[ip], TokenTypes.Equals);if (ip+1>(chars.length-1)){EOF=1; break;}else{ip++;}}
+            if (chars[ip] == "+"){tokens=BuildToken(tokens, chars[ip], TokenTypes.BinaryExpression);if (ip+1>(chars.length-1)){EOF=1; break;}else{ip++;}}
+            if (chars[ip] == "-"){tokens=BuildToken(tokens, chars[ip], TokenTypes.BinaryExpression);if (ip+1>(chars.length-1)){EOF=1; break;}else{ip++;}}
+            if (chars[ip] == "*"){tokens=BuildToken(tokens, chars[ip], TokenTypes.BinaryExpression);if (ip+1>(chars.length-1)){EOF=1; break;}else{ip++;}}
+            if (chars[ip] == "%"){tokens=BuildToken(tokens, chars[ip], TokenTypes.BinaryExpression);if (ip+1>(chars.length-1)){EOF=1; break;}else{ip++;}}
+            if (chars[ip] == "/"){tokens=BuildToken(tokens, chars[ip], TokenTypes.BinaryExpression);if (ip+1>(chars.length-1)){EOF=1; break;}else{ip++;}}
+            
+        }else{
+        
+            if (IsNumerical(chars[ip])){
+
+                let Number = "";
+                while (IsNumerical(chars[ip])){
+                    
+                    Number += chars[ip];
+                    if (ip+1>(chars.length-1)){EOF=1; break;}else{ip++;}
+
+                }
+                tokens = BuildToken(tokens, Number, TokenTypes.Numeral);
+
+            }
+
+            if (IsAlphabetical(chars[ip])){
+
+                let String = "";
+                while (IsAlphabetical(chars[ip])){
+                    String += chars[ip];
+                    if (ip+1>(chars.length-1)){EOF=1; break;}else{ip++;}
+                }
+                const tf = IsKeyword(String)[0];
+                const k = IsKeyword(String)[1];
+                if (tf){
+                    tokens = BuildToken(tokens, String, k);
+                }else{
+                    tokens = BuildToken(tokens, String, TokenTypes.Identifier);
+                }
+
+            }
+
+
+            if (IsSkippable(chars[ip])){
+                if (ip+1>(chars.length-1)){EOF=1; break;}else{ip++;}
+            }
+
+            if (!IsAlphabetical(chars[ip]) == !IsNumerical(chars[ip]) == !IsOneCharToken(chars[ip]) == !IsSkippable(chars[ip])){
+                console.log("Char That Cannot Be Handeled found in src -> { "+chars[ip]+" }");
+                Deno.exit(2);
+            }
+
+        }
+
+    }
+
+    tokens = BuildToken(tokens, "EOF", TokenTypes.EOF)
+
+    return tokens;
+
+}