diff --git a/compiler/src/main/scala/Main.scala b/compiler/src/main/scala/Main.scala index 1678a03..d26efa1 100644 --- a/compiler/src/main/scala/Main.scala +++ b/compiler/src/main/scala/Main.scala @@ -1,10 +1,13 @@ package compiler +import scala.util.{Try, Failure, Success} + import java.io.{BufferedReader, InputStreamReader, PrintWriter} import java.net.{ServerSocket, Socket}; import java.util.stream.Stream import compiler.Tokenizer._ +import compiler.Parser._ @main def hello(): Unit = @@ -45,4 +48,9 @@ def run_server(): Unit = out.println(response); def compile(source: String): String = - return Tokenizer.tokenize(source).toString() + val tokens = Tokenizer.tokenize(source) + val ast = tokens match { + case Success(tokens) => Parser.parse(tokens) + case Failure(err) => Failure(err) + } + return ast.toString() diff --git a/compiler/src/main/scala/Parser.scala b/compiler/src/main/scala/Parser.scala new file mode 100644 index 0000000..14c354c --- /dev/null +++ b/compiler/src/main/scala/Parser.scala @@ -0,0 +1,32 @@ +package compiler.Parser + +import scala.util.{Try, Success, Failure} + +import compiler.Tokenizer.{Token, TokenType} + +sealed trait Expression: + //override def toString() = s"${this.getClass.getName.stripPrefix("compiler.Parser.")}($this)" + override def toString() = s"Expression(TODO)" + + +class Literal(value: Int | Boolean) extends Expression +class Identifier(name: String) extends Expression +class BinaryOperator(left: Expression, operator: String, right: Expression) extends Expression +class Other() extends Expression + +class Parser(tokens: List[Token]): + // Keeps track of current token, index to tokens: List[Token] + private var pos = 0 + + def peek(): Token = + return tokens(pos) + + def consume(expected: TokenType | List[TokenType]): Token = + val token = peek() + + pos += 1 + return tokens(pos) + + +object Parser: + def parse(tokens: List[Token]): Try[Expression] = return Success(Other()) diff --git a/compiler/src/main/scala/Tokenizer.scala b/compiler/src/main/scala/Tokenizer.scala index fb1dedf..716b1b1 100644 --- a/compiler/src/main/scala/Tokenizer.scala +++ b/compiler/src/main/scala/Tokenizer.scala @@ -30,7 +30,14 @@ class TokenLocationDebug(): case _ => false } -sealed trait Token(val value: String, val location: TokenLocation | TokenLocationDebug = TokenLocationDebug()): +// Try to keep this in sync with the above classes +enum TokenType: + case Whitespace, Comment, IntLiteral, Operator, Punctuation, Identifier, Other, Error + +// Access TokenType cases without TokenType. prefix: TokenType.Identifier vs Identifier +import compiler.Tokenizer.TokenType._ + +class Token(val value: String, val tokenType: TokenType ,val location: TokenLocation | TokenLocationDebug): override def toString: String = s"${this.getClass.getName.stripPrefix("compiler.Tokenizer.")}($value, $location)" override def equals(other: Any): Boolean = other match { case that: Token => { @@ -42,16 +49,14 @@ sealed trait Token(val value: String, val location: TokenLocation | TokenLocatio case _ => false } def isSkippable() = - this match { - case that: WhitespaceT => true - case that: CommentT => true + this.tokenType match { + case Whitespace => true + case Comment => true case _ => false } object Token: def tokenFromString(str: String, currentPos: Position): Option[Token] = - - if Token.isWhitespace(str) then val startWhitespace = Position(currentPos.line, currentPos.column) str.foreach(_ match { @@ -62,7 +67,7 @@ object Token: val endWhitespace = Position(currentPos.line, currentPos.column) val tokenLoc = TokenLocation(startWhitespace, endWhitespace) - return Some(WhitespaceT(str, tokenLoc)) + return Some(Token(str, Whitespace, tokenLoc)) // After whitespace is calculated, move currentPos to the end of the current token // i.e. current column + token length @@ -71,11 +76,11 @@ object Token: val end = Position(currentPos.line, currentPos.column) val tokenLoc = TokenLocation(start, end) - if Token.isComment(str) then return Some(CommentT(str, tokenLoc)) - if Token.isIntLiteral(str) then return Some(IntLiteralT(str, tokenLoc)) - if Token.isOperator(str) then return Some(OperatorT(str, tokenLoc)) - if Token.isIdentifier(str) then return Some(IdentifierT(str, tokenLoc)) - if Token.isPunctuation(str) then return Some(PunctuationT(str, tokenLoc)) + if Token.isComment(str) then return Some(Token(str, Comment, tokenLoc)) + if Token.isIntLiteral(str) then return Some(Token(str, IntLiteral, tokenLoc)) + if Token.isOperator(str) then return Some(Token(str, Operator, tokenLoc)) + if Token.isIdentifier(str) then return Some(Token(str, Identifier, tokenLoc)) + if Token.isPunctuation(str) then return Some(Token(str, Punctuation, tokenLoc)) else return None @@ -86,13 +91,6 @@ object Token: def isPunctuation(str: String): Boolean = raw"[(),;:{}]".r.matches(str) def isIdentifier(str: String): Boolean = raw"[a-zA-Z_][a-zA-Z_0-9]*".r.matches(str) -class WhitespaceT(value: String, location: TokenLocation | TokenLocationDebug) extends Token(value, location) -class CommentT(value: String, location: TokenLocation | TokenLocationDebug) extends Token(value, location) -class IntLiteralT(value: String, location: TokenLocation | TokenLocationDebug) extends Token(value, location) -class OperatorT(value: String, location: TokenLocation | TokenLocationDebug) extends Token(value, location) -class PunctuationT(value: String, location: TokenLocation | TokenLocationDebug) extends Token(value, location) -class IdentifierT(value: String, location: TokenLocation | TokenLocationDebug) extends Token(value, location) -class OtherT(value: String, location: TokenLocation | TokenLocationDebug) extends Token(value, location) object Tokenizer: diff --git a/compiler/src/test/scala/MySuite.scala b/compiler/src/test/scala/MySuite.scala index a84d315..91d6ab1 100644 --- a/compiler/src/test/scala/MySuite.scala +++ b/compiler/src/test/scala/MySuite.scala @@ -1,7 +1,8 @@ // For more information on writing tests, see // https://scalameta.org/munit/docs/getting-started.html -import compiler.Tokenizer.* +import compiler.Tokenizer._ +import compiler.Tokenizer.TokenType._ import scala.util.{Try, Failure, Success} @@ -25,11 +26,11 @@ class TokenizerTestWithoutLocation extends munit.FunSuite { test("should work with identifiers") { val code = "while true do print_int BigInt" val expected = List( - IdentifierT("while", L), - IdentifierT("true", L), - IdentifierT("do", L), - IdentifierT("print_int", L), - IdentifierT("BigInt", L), + Token("while", Identifier, L), + Token("true", Identifier, L), + Token("do", Identifier, L), + Token("print_int", Identifier, L), + Token("BigInt", Identifier, L), ) test_tokenizer(code, expected) } @@ -38,11 +39,11 @@ class TokenizerTestWithoutLocation extends munit.FunSuite { val code = "41 + 3 == 44" val expected = List( - IntLiteralT("41", L), - OperatorT("+", L), - IntLiteralT("3", L), - OperatorT("==", L), - IntLiteralT("44", L) + Token("41", IntLiteral, L), + Token("+", Operator, L), + Token("3", IntLiteral, L), + Token("==", Operator, L), + Token("44", IntLiteral, L) ) test_tokenizer(code, expected) } @@ -50,7 +51,13 @@ class TokenizerTestWithoutLocation extends munit.FunSuite { test("should work with operators and int literals") { val code = "1531 + 1241 == 2772" val expected = - List(IntLiteralT("1531", L), OperatorT("+", L), IntLiteralT("1241", L), OperatorT("==", L), IntLiteralT("2772", L)) + List( + Token("1531", IntLiteral, L), + Token("+", Operator, L), + Token("1241", IntLiteral, L), + Token("==", Operator, L), + Token("2772", IntLiteral, L) + ) test_tokenizer(code, expected) } @@ -59,11 +66,11 @@ class TokenizerTestWithoutLocation extends munit.FunSuite { val code2 = "4/2==2 // math" val code3 = "# this is a whole line comment" val expected = List( - IntLiteralT("4", L), - OperatorT("/", L), - IntLiteralT("2", L), - OperatorT("==", L), - IntLiteralT("2", L), + Token("4", IntLiteral, L), + Token("/", Operator, L), + Token("2", IntLiteral, L), + Token("==", Operator, L), + Token("2", IntLiteral, L), ) test_tokenizer(code1, expected) test_tokenizer(code2, expected) @@ -80,11 +87,11 @@ class TokenizerTestWithoutLocation extends munit.FunSuite { test("should work with function parenthesis") { val code = "print_int(n);" val expected = List( - IdentifierT("print_int", L), - PunctuationT("(", L), - IdentifierT("n", L), - PunctuationT(")", L), - PunctuationT(";", L), + Token("print_int", Identifier, L), + Token("(", Punctuation, L), + Token("n", Identifier, L), + Token(")", Punctuation, L), + Token(";", Punctuation, L), ) test_tokenizer(code, expected) } @@ -92,17 +99,17 @@ class TokenizerTestWithoutLocation extends munit.FunSuite { test("should work with functions with types") { val code = "test(i: Int,j: Float);" val expected = List( - IdentifierT("test", L), - PunctuationT("(", L), - IdentifierT("i", L), - PunctuationT(":", L), - IdentifierT("Int", L), - PunctuationT(",", L), - IdentifierT("j", L), - PunctuationT(":", L), - IdentifierT("Float", L), - PunctuationT(")", L), - PunctuationT(";", L), + Token("test", Identifier, L), + Token("(", Punctuation, L), + Token("i", Identifier, L), + Token(":", Punctuation, L), + Token("Int", Identifier, L), + Token(",", Punctuation, L), + Token("j", Identifier, L), + Token(":", Punctuation, L), + Token("Float", Identifier, L), + Token(")", Punctuation, L), + Token(";", Punctuation, L), ) test_tokenizer(code, expected) } @@ -110,18 +117,18 @@ class TokenizerTestWithoutLocation extends munit.FunSuite { test("should work with more complicated control flow") { val code = "if (i>0) { it(); }" val expected = List( - IdentifierT("if", L), - PunctuationT("(", L), - IdentifierT("i", L), - OperatorT(">", L), - IntLiteralT("0", L), - PunctuationT(")", L), - PunctuationT("{", L), - IdentifierT("it", L), - PunctuationT("(", L), - PunctuationT(")", L), - PunctuationT(";", L), - PunctuationT("}", L), + Token("if", Identifier, L), + Token("(", Punctuation, L), + Token("i", Identifier, L), + Token(">", Operator, L), + Token("0", IntLiteral, L), + Token(")", Punctuation, L), + Token("{", Punctuation, L), + Token("it", Identifier, L), + Token("(", Punctuation, L), + Token(")", Punctuation, L), + Token(";", Punctuation, L), + Token("}", Punctuation, L), ) test_tokenizer(code, expected) } @@ -139,7 +146,7 @@ class TokenizerTestWithLocation extends munit.FunSuite { test("should fail with incorrect positions") { val code = "1453" val location = TokenLocation(Position(2, 1), Position(4,10)) - val expected = List(IntLiteralT(code, location)) + val expected = List(Token(code, IntLiteral, location)) val result = Tokenizer.tokenize(code) result match { case Success(result) => assertNotEquals(result, expected) @@ -150,16 +157,16 @@ class TokenizerTestWithLocation extends munit.FunSuite { test("should work with int literals") { val code = "1453" val location = TokenLocation(Position(1, 1), Position(1,5)) - val expected = List(IntLiteralT(code, location)) + val expected = List(Token(code, IntLiteral, location)) test_tokenizer(code, expected) } test("should work with a newline") { val code = "if 3\nwhile" val expected = List( - IdentifierT("if", TokenLocation(Position(1,1), Position(1,3))), - IntLiteralT("3", TokenLocation(Position(1,5), Position(1,6))), - IdentifierT("while", TokenLocation(Position(2,1), Position(2,6))) + Token("if", Identifier, TokenLocation(Position(1,1), Position(1,3))), + Token("3", IntLiteral, TokenLocation(Position(1,5), Position(1,6))), + Token("while", Identifier, TokenLocation(Position(2,1), Position(2,6))) ) test_tokenizer(code, expected) } @@ -167,10 +174,10 @@ class TokenizerTestWithLocation extends munit.FunSuite { test("should work with multiple newline") { val code = "if 3\nwhile\n\nprint_int" val expected = List( - IdentifierT("if", TokenLocation(Position(1,1), Position(1,3))), - IntLiteralT("3", TokenLocation(Position(1,5), Position(1,6))), - IdentifierT("while", TokenLocation(Position(2,1), Position(2,6))), - IdentifierT("print_int", TokenLocation(Position(4,1), Position(4,10))) + Token("if", Identifier, TokenLocation(Position(1,1), Position(1,3))), + Token("3", IntLiteral, TokenLocation(Position(1,5), Position(1,6))), + Token("while", Identifier, TokenLocation(Position(2,1), Position(2,6))), + Token("print_int", Identifier, TokenLocation(Position(4,1), Position(4,10))) ) test_tokenizer(code, expected) } @@ -178,17 +185,17 @@ class TokenizerTestWithLocation extends munit.FunSuite { test("should work with multiple tokens") { val code = "1 + 2 % 4 / 3 * 3 <= 100" val expected = List( - IntLiteralT("1", TokenLocation(Position(1,1), Position(1,2))), - OperatorT("+", TokenLocation(Position(1,3), Position(1,4))), - IntLiteralT("2", TokenLocation(Position(1,5), Position(1,6))), - OperatorT("%", TokenLocation(Position(1,7), Position(1,8))), - IntLiteralT("4", TokenLocation(Position(1,9), Position(1,10))), - OperatorT("/", TokenLocation(Position(1,11), Position(1,12))), - IntLiteralT("3", TokenLocation(Position(1,13), Position(1,14))), - OperatorT("*", TokenLocation(Position(1,15), Position(1,16))), - IntLiteralT("3", TokenLocation(Position(1,17), Position(1,18))), - OperatorT("<=", TokenLocation(Position(1,19), Position(1,21))), - IntLiteralT("100", TokenLocation(Position(1,22), Position(1,25))) + Token("1", IntLiteral, TokenLocation(Position(1,1), Position(1,2))), + Token("+", Operator, TokenLocation(Position(1,3), Position(1,4))), + Token("2", IntLiteral, TokenLocation(Position(1,5), Position(1,6))), + Token("%", Operator, TokenLocation(Position(1,7), Position(1,8))), + Token("4", IntLiteral, TokenLocation(Position(1,9), Position(1,10))), + Token("/", Operator, TokenLocation(Position(1,11), Position(1,12))), + Token("3", IntLiteral, TokenLocation(Position(1,13), Position(1,14))), + Token("*", Operator, TokenLocation(Position(1,15), Position(1,16))), + Token("3", IntLiteral, TokenLocation(Position(1,17), Position(1,18))), + Token("<=", Operator, TokenLocation(Position(1,19), Position(1,21))), + Token("100", IntLiteral, TokenLocation(Position(1,22), Position(1,25))) ) test_tokenizer(code, expected) }