Refactored Token class and added initial ParseR

compiler/src/main/scala/Main.scala

@@ -1,10 +1,13 @@
 package compiler
 
+import scala.util.{Try, Failure, Success}
+
 import java.io.{BufferedReader, InputStreamReader, PrintWriter}
 import java.net.{ServerSocket, Socket};
 import java.util.stream.Stream
 
 import compiler.Tokenizer._
+import compiler.Parser._
 
 
 @main def hello(): Unit =
@@ -45,4 +48,9 @@ def run_server(): Unit =
     out.println(response);
 
 def compile(source: String): String =
-  return Tokenizer.tokenize(source).toString()
+  val tokens = Tokenizer.tokenize(source)
+  val ast = tokens match {
+    case Success(tokens) => Parser.parse(tokens)
+    case Failure(err) => Failure(err)
+  }
+  return ast.toString()

compiler/src/main/scala/Parser.scala

@@ -0,0 +1,32 @@
+package compiler.Parser
+
+import scala.util.{Try, Success, Failure}
+
+import compiler.Tokenizer.{Token, TokenType}
+
+sealed trait Expression:
+  //override def toString() = s"${this.getClass.getName.stripPrefix("compiler.Parser.")}($this)"
+  override def toString() = s"Expression(TODO)"
+
+
+class Literal(value: Int | Boolean) extends Expression
+class Identifier(name: String) extends Expression
+class BinaryOperator(left: Expression, operator: String, right: Expression) extends Expression
+class Other() extends Expression
+
+class Parser(tokens: List[Token]):
+  // Keeps track of current token, index to tokens: List[Token]
+  private var pos = 0
+
+  def peek(): Token =
+    return tokens(pos)
+
+  def consume(expected: TokenType | List[TokenType]): Token =
+    val token = peek()
+    
+    pos += 1
+    return tokens(pos)
+
+
+object Parser:
+  def parse(tokens: List[Token]): Try[Expression] = return Success(Other())

compiler/src/main/scala/Tokenizer.scala

@@ -30,7 +30,14 @@ class TokenLocationDebug():
     case _ => false
   }
 
-sealed trait Token(val value: String, val location: TokenLocation | TokenLocationDebug = TokenLocationDebug()):
+// Try to keep this in sync with the above classes
+enum TokenType:
+  case Whitespace, Comment, IntLiteral, Operator, Punctuation, Identifier, Other, Error
+
+// Access TokenType cases without TokenType. prefix: TokenType.Identifier vs Identifier
+import compiler.Tokenizer.TokenType._
+
+class Token(val value: String, val tokenType: TokenType ,val location: TokenLocation | TokenLocationDebug):
   override def toString: String = s"${this.getClass.getName.stripPrefix("compiler.Tokenizer.")}($value, $location)"
   override def equals(other: Any): Boolean = other match {
     case that: Token => {
@@ -42,16 +49,14 @@ sealed trait Token(val value: String, val location: TokenLocation | TokenLocatio
     case _ => false
   }
   def isSkippable() =
-    this match {
+    this.tokenType match {
-      case that: WhitespaceT => true
+      case Whitespace => true
-      case that: CommentT => true
+      case Comment => true
       case _ => false
     }
 
 object Token:
   def tokenFromString(str: String, currentPos: Position): Option[Token] =
-
-
     if Token.isWhitespace(str) then 
       val startWhitespace = Position(currentPos.line, currentPos.column)
       str.foreach(_ match {
@@ -62,7 +67,7 @@ object Token:
 
       val endWhitespace = Position(currentPos.line, currentPos.column)
       val tokenLoc = TokenLocation(startWhitespace, endWhitespace)
-      return Some(WhitespaceT(str, tokenLoc))
+      return Some(Token(str, Whitespace, tokenLoc))
 
     // After whitespace is calculated, move currentPos to the end of the current token
     // i.e. current column + token length
@@ -71,11 +76,11 @@ object Token:
     val end = Position(currentPos.line, currentPos.column)
     val tokenLoc = TokenLocation(start, end)
 
-    if Token.isComment(str) then return Some(CommentT(str, tokenLoc))
+    if Token.isComment(str) then return Some(Token(str, Comment, tokenLoc))
-    if Token.isIntLiteral(str) then return Some(IntLiteralT(str, tokenLoc))
+    if Token.isIntLiteral(str) then return Some(Token(str, IntLiteral, tokenLoc))
-    if Token.isOperator(str) then return Some(OperatorT(str, tokenLoc))
+    if Token.isOperator(str) then return Some(Token(str, Operator, tokenLoc))
-    if Token.isIdentifier(str) then return Some(IdentifierT(str, tokenLoc))
+    if Token.isIdentifier(str) then return Some(Token(str, Identifier, tokenLoc))
-    if Token.isPunctuation(str) then return Some(PunctuationT(str, tokenLoc))
+    if Token.isPunctuation(str) then return Some(Token(str, Punctuation, tokenLoc))
     else return None
 
 
@@ -86,13 +91,6 @@ object Token:
   def isPunctuation(str: String): Boolean = raw"[(),;:{}]".r.matches(str)
   def isIdentifier(str: String): Boolean = raw"[a-zA-Z_][a-zA-Z_0-9]*".r.matches(str)
 
-class WhitespaceT(value: String, location: TokenLocation | TokenLocationDebug) extends Token(value, location)
-class CommentT(value: String, location: TokenLocation | TokenLocationDebug) extends Token(value, location)
-class IntLiteralT(value: String, location: TokenLocation | TokenLocationDebug) extends Token(value, location)
-class OperatorT(value: String, location: TokenLocation | TokenLocationDebug) extends Token(value, location)
-class PunctuationT(value: String, location: TokenLocation | TokenLocationDebug) extends Token(value, location)
-class IdentifierT(value: String, location: TokenLocation | TokenLocationDebug) extends Token(value, location)
-class OtherT(value: String, location: TokenLocation | TokenLocationDebug) extends Token(value, location)
 
 
 object Tokenizer:

compiler/src/test/scala/MySuite.scala

@@ -1,7 +1,8 @@
 // For more information on writing tests, see
 // https://scalameta.org/munit/docs/getting-started.html
 
-import compiler.Tokenizer.*
+import compiler.Tokenizer._
+import compiler.Tokenizer.TokenType._
 import scala.util.{Try, Failure, Success}
 
 
@@ -25,11 +26,11 @@ class TokenizerTestWithoutLocation extends munit.FunSuite {
   test("should work with identifiers") {
     val code = "while true do print_int BigInt"
     val expected = List(
-      IdentifierT("while", L),
+      Token("while", Identifier, L),
-      IdentifierT("true", L),
+      Token("true", Identifier, L),
-      IdentifierT("do", L),
+      Token("do", Identifier, L),
-      IdentifierT("print_int", L),
+      Token("print_int", Identifier, L),
-      IdentifierT("BigInt", L),
+      Token("BigInt", Identifier, L),
     )
     test_tokenizer(code, expected)
   }
@@ -38,11 +39,11 @@ class TokenizerTestWithoutLocation extends munit.FunSuite {
     val code = "41 + 3 == 44"
     val expected =
       List(
-        IntLiteralT("41", L),
+        Token("41", IntLiteral, L),
-        OperatorT("+", L),
+        Token("+", Operator, L),
-        IntLiteralT("3", L),
+        Token("3", IntLiteral, L),
-        OperatorT("==", L),
+        Token("==", Operator, L),
-        IntLiteralT("44", L)
+        Token("44", IntLiteral, L)
       )
     test_tokenizer(code, expected)
   }
@@ -50,7 +51,13 @@ class TokenizerTestWithoutLocation extends munit.FunSuite {
   test("should work with operators and int literals") {
     val code = "1531 + 1241 == 2772"
     val expected =
-      List(IntLiteralT("1531", L), OperatorT("+", L), IntLiteralT("1241", L), OperatorT("==", L), IntLiteralT("2772", L))
+      List(
+        Token("1531", IntLiteral, L), 
+        Token("+", Operator, L),
+        Token("1241", IntLiteral, L),
+        Token("==", Operator, L),
+        Token("2772", IntLiteral, L)
+      )
     test_tokenizer(code, expected)
   }
 
@@ -59,11 +66,11 @@ class TokenizerTestWithoutLocation extends munit.FunSuite {
     val code2 = "4/2==2 // math"
     val code3 = "# this is a whole line comment"
     val expected = List(
-      IntLiteralT("4", L),
+      Token("4", IntLiteral, L),
-      OperatorT("/", L),
+      Token("/", Operator, L),
-      IntLiteralT("2", L),
+      Token("2", IntLiteral, L),
-      OperatorT("==", L),
+      Token("==", Operator, L),
-      IntLiteralT("2", L),
+      Token("2", IntLiteral, L),
     )
     test_tokenizer(code1, expected)
     test_tokenizer(code2, expected)
@@ -80,11 +87,11 @@ class TokenizerTestWithoutLocation extends munit.FunSuite {
   test("should work with function parenthesis") {
     val code = "print_int(n);"
     val expected = List(
-      IdentifierT("print_int", L),
+      Token("print_int", Identifier, L),
-      PunctuationT("(", L),
+      Token("(", Punctuation, L),
-      IdentifierT("n", L),
+      Token("n", Identifier, L),
-      PunctuationT(")", L),
+      Token(")", Punctuation, L),
-      PunctuationT(";", L),
+      Token(";", Punctuation, L),
     )
     test_tokenizer(code, expected)
   }
@@ -92,17 +99,17 @@ class TokenizerTestWithoutLocation extends munit.FunSuite {
   test("should work with functions with types") {
     val code = "test(i: Int,j: Float);"
     val expected = List(
-      IdentifierT("test", L),
+      Token("test", Identifier, L),
-      PunctuationT("(", L),
+      Token("(", Punctuation, L),
-      IdentifierT("i", L),
+      Token("i", Identifier, L),
-      PunctuationT(":", L),
+      Token(":", Punctuation, L),
-      IdentifierT("Int", L),
+      Token("Int", Identifier, L),
-      PunctuationT(",", L),
+      Token(",", Punctuation, L),
-      IdentifierT("j", L),
+      Token("j", Identifier, L),
-      PunctuationT(":", L),
+      Token(":", Punctuation, L),
-      IdentifierT("Float", L),
+      Token("Float", Identifier, L),
-      PunctuationT(")", L),
+      Token(")", Punctuation, L),
-      PunctuationT(";", L),
+      Token(";", Punctuation, L),
     )
     test_tokenizer(code, expected)
   }
@@ -110,18 +117,18 @@ class TokenizerTestWithoutLocation extends munit.FunSuite {
   test("should work with more complicated control flow") {
     val code = "if (i>0) { it(); }"
     val expected = List(
-      IdentifierT("if", L),
+      Token("if", Identifier, L),
-      PunctuationT("(", L),
+      Token("(", Punctuation, L),
-      IdentifierT("i", L),
+      Token("i", Identifier, L),
-      OperatorT(">", L),
+      Token(">", Operator, L),
-      IntLiteralT("0", L),
+      Token("0", IntLiteral, L),
-      PunctuationT(")", L),
+      Token(")", Punctuation, L),
-      PunctuationT("{", L),
+      Token("{", Punctuation, L),
-      IdentifierT("it", L),
+      Token("it", Identifier, L),
-      PunctuationT("(", L),
+      Token("(", Punctuation, L),
-      PunctuationT(")", L),
+      Token(")", Punctuation, L),
-      PunctuationT(";", L),
+      Token(";", Punctuation, L),
-      PunctuationT("}", L),
+      Token("}", Punctuation, L),
     )
     test_tokenizer(code, expected)
   }
@@ -139,7 +146,7 @@ class TokenizerTestWithLocation extends munit.FunSuite {
   test("should fail with incorrect positions") {
     val code = "1453"
     val location = TokenLocation(Position(2, 1), Position(4,10))
-    val expected = List(IntLiteralT(code, location))
+    val expected = List(Token(code, IntLiteral, location))
     val result = Tokenizer.tokenize(code)
     result match {
       case Success(result) => assertNotEquals(result, expected)
@@ -150,16 +157,16 @@ class TokenizerTestWithLocation extends munit.FunSuite {
   test("should work with int literals") {
     val code = "1453"
     val location = TokenLocation(Position(1, 1), Position(1,5))
-    val expected = List(IntLiteralT(code, location))
+    val expected = List(Token(code, IntLiteral, location))
     test_tokenizer(code, expected)
   }
 
   test("should work with a newline") {
     val code = "if  3\nwhile"
     val expected = List(
-      IdentifierT("if", TokenLocation(Position(1,1), Position(1,3))),
+      Token("if", Identifier, TokenLocation(Position(1,1), Position(1,3))),
-      IntLiteralT("3", TokenLocation(Position(1,5), Position(1,6))),
+      Token("3", IntLiteral, TokenLocation(Position(1,5), Position(1,6))),
-      IdentifierT("while", TokenLocation(Position(2,1), Position(2,6)))
+      Token("while", Identifier, TokenLocation(Position(2,1), Position(2,6)))
     )
     test_tokenizer(code, expected)
   }
@@ -167,10 +174,10 @@ class TokenizerTestWithLocation extends munit.FunSuite {
   test("should work with multiple newline") {
     val code = "if  3\nwhile\n\nprint_int"
     val expected = List(
-      IdentifierT("if", TokenLocation(Position(1,1), Position(1,3))),
+      Token("if", Identifier, TokenLocation(Position(1,1), Position(1,3))),
-      IntLiteralT("3", TokenLocation(Position(1,5), Position(1,6))),
+      Token("3", IntLiteral, TokenLocation(Position(1,5), Position(1,6))),
-      IdentifierT("while", TokenLocation(Position(2,1), Position(2,6))),
+      Token("while", Identifier, TokenLocation(Position(2,1), Position(2,6))),
-      IdentifierT("print_int", TokenLocation(Position(4,1), Position(4,10)))
+      Token("print_int", Identifier, TokenLocation(Position(4,1), Position(4,10)))
     )
     test_tokenizer(code, expected)
   }
@@ -178,17 +185,17 @@ class TokenizerTestWithLocation extends munit.FunSuite {
   test("should work with multiple tokens") {
     val code = "1 + 2 % 4 / 3 * 3 <= 100"
     val expected = List(
-      IntLiteralT("1", TokenLocation(Position(1,1), Position(1,2))),
+      Token("1", IntLiteral, TokenLocation(Position(1,1), Position(1,2))),
-      OperatorT("+", TokenLocation(Position(1,3), Position(1,4))),
+      Token("+", Operator, TokenLocation(Position(1,3), Position(1,4))),
-      IntLiteralT("2", TokenLocation(Position(1,5), Position(1,6))),
+      Token("2", IntLiteral, TokenLocation(Position(1,5), Position(1,6))),
-      OperatorT("%", TokenLocation(Position(1,7), Position(1,8))),
+      Token("%", Operator, TokenLocation(Position(1,7), Position(1,8))),
-      IntLiteralT("4", TokenLocation(Position(1,9), Position(1,10))),
+      Token("4", IntLiteral, TokenLocation(Position(1,9), Position(1,10))),
-      OperatorT("/", TokenLocation(Position(1,11), Position(1,12))),
+      Token("/", Operator, TokenLocation(Position(1,11), Position(1,12))),
-      IntLiteralT("3", TokenLocation(Position(1,13), Position(1,14))),
+      Token("3", IntLiteral, TokenLocation(Position(1,13), Position(1,14))),
-      OperatorT("*", TokenLocation(Position(1,15), Position(1,16))),
+      Token("*", Operator, TokenLocation(Position(1,15), Position(1,16))),
-      IntLiteralT("3", TokenLocation(Position(1,17), Position(1,18))),
+      Token("3", IntLiteral, TokenLocation(Position(1,17), Position(1,18))),
-      OperatorT("<=", TokenLocation(Position(1,19), Position(1,21))),
+      Token("<=", Operator, TokenLocation(Position(1,19), Position(1,21))),
-      IntLiteralT("100", TokenLocation(Position(1,22), Position(1,25)))
+      Token("100", IntLiteral, TokenLocation(Position(1,22), Position(1,25)))
     )
     test_tokenizer(code, expected)
   }