iritmaximus/compilers
1
0
Fork 0
mirror of https://github.com/iritmaximus/compilers.git synced 2026-10-12 19:13:35 +03:00
Code Issues Packages Projects Releases Wiki Activity

Refactored Token class and added initial ParseR

Browse Source
This commit is contained in:
Aaro Koivukangas
2026-02-01 22:32:23 +02:00
parent 2249a187f0
commit 67a82cfc6b
4 changed files with 130 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files
compiler/src/main/scala/Main.scala
+9 -1
View File
@@ -1,10 +1,13 @@
package compiler
import scala.util.{Try, Failure, Success}
import java.io.{BufferedReader, InputStreamReader, PrintWriter}
import java.net.{ServerSocket, Socket};
import java.util.stream.Stream
import compiler.Tokenizer._
import compiler.Parser._
@main def hello(): Unit =
@@ -45,4 +48,9 @@ def run_server(): Unit =
out.println(response);
def compile(source: String): String =
return Tokenizer.tokenize(source).toString()
val tokens = Tokenizer.tokenize(source)
val ast = tokens match {
case Success(tokens) => Parser.parse(tokens)
case Failure(err) => Failure(err)
}
return ast.toString()
compiler/src/main/scala/Parser.scala
+32
View File
@@ -0,0 +1,32 @@
package compiler.Parser
import scala.util.{Try, Success, Failure}
import compiler.Tokenizer.{Token, TokenType}
sealed trait Expression:
//override def toString() = s"${this.getClass.getName.stripPrefix("compiler.Parser.")}($this)"
override def toString() = s"Expression(TODO)"
class Literal(value: Int | Boolean) extends Expression
class Identifier(name: String) extends Expression
class BinaryOperator(left: Expression, operator: String, right: Expression) extends Expression
class Other() extends Expression
class Parser(tokens: List[Token]):
// Keeps track of current token, index to tokens: List[Token]
private var pos = 0
def peek(): Token =
return tokens(pos)
def consume(expected: TokenType | List[TokenType]): Token =
val token = peek()
pos += 1
return tokens(pos)
object Parser:
def parse(tokens: List[Token]): Try[Expression] = return Success(Other())
compiler/src/main/scala/Tokenizer.scala
+17 -19
View File
@@ -30,7 +30,14 @@ class TokenLocationDebug():
case _ => false
}
sealed trait Token(val value: String, val location: TokenLocation | TokenLocationDebug = TokenLocationDebug()):
// Try to keep this in sync with the above classes
enum TokenType:
case Whitespace, Comment, IntLiteral, Operator, Punctuation, Identifier, Other, Error
// Access TokenType cases without TokenType. prefix: TokenType.Identifier vs Identifier
import compiler.Tokenizer.TokenType._
class Token(val value: String, val tokenType: TokenType ,val location: TokenLocation | TokenLocationDebug):
override def toString: String = s"${this.getClass.getName.stripPrefix("compiler.Tokenizer.")}($value, $location)"
override def equals(other: Any): Boolean = other match {
case that: Token => {
@@ -42,16 +49,14 @@ sealed trait Token(val value: String, val location: TokenLocation | TokenLocatio
case _ => false
}
def isSkippable() =
this match {
case that: WhitespaceT => true
case that: CommentT => true
this.tokenType match {
case Whitespace => true
case Comment => true
case _ => false
}
object Token:
def tokenFromString(str: String, currentPos: Position): Option[Token] =
if Token.isWhitespace(str) then
val startWhitespace = Position(currentPos.line, currentPos.column)
str.foreach(_ match {
@@ -62,7 +67,7 @@ object Token:
val endWhitespace = Position(currentPos.line, currentPos.column)
val tokenLoc = TokenLocation(startWhitespace, endWhitespace)
return Some(WhitespaceT(str, tokenLoc))
return Some(Token(str, Whitespace, tokenLoc))
// After whitespace is calculated, move currentPos to the end of the current token
// i.e. current column + token length
@@ -71,11 +76,11 @@ object Token:
val end = Position(currentPos.line, currentPos.column)
val tokenLoc = TokenLocation(start, end)
if Token.isComment(str) then return Some(CommentT(str, tokenLoc))
if Token.isIntLiteral(str) then return Some(IntLiteralT(str, tokenLoc))
if Token.isOperator(str) then return Some(OperatorT(str, tokenLoc))
if Token.isIdentifier(str) then return Some(IdentifierT(str, tokenLoc))
if Token.isPunctuation(str) then return Some(PunctuationT(str, tokenLoc))
if Token.isComment(str) then return Some(Token(str, Comment, tokenLoc))
if Token.isIntLiteral(str) then return Some(Token(str, IntLiteral, tokenLoc))
if Token.isOperator(str) then return Some(Token(str, Operator, tokenLoc))
if Token.isIdentifier(str) then return Some(Token(str, Identifier, tokenLoc))
if Token.isPunctuation(str) then return Some(Token(str, Punctuation, tokenLoc))
else return None
@@ -86,13 +91,6 @@ object Token:
def isPunctuation(str: String): Boolean = raw"[(),;:{}]".r.matches(str)
def isIdentifier(str: String): Boolean = raw"[a-zA-Z_][a-zA-Z_0-9]*".r.matches(str)
class WhitespaceT(value: String, location: TokenLocation | TokenLocationDebug) extends Token(value, location)
class CommentT(value: String, location: TokenLocation | TokenLocationDebug) extends Token(value, location)
class IntLiteralT(value: String, location: TokenLocation | TokenLocationDebug) extends Token(value, location)
class OperatorT(value: String, location: TokenLocation | TokenLocationDebug) extends Token(value, location)
class PunctuationT(value: String, location: TokenLocation | TokenLocationDebug) extends Token(value, location)
class IdentifierT(value: String, location: TokenLocation | TokenLocationDebug) extends Token(value, location)
class OtherT(value: String, location: TokenLocation | TokenLocationDebug) extends Token(value, location)
object Tokenizer:
compiler/src/test/scala/MySuite.scala
+72 -65
View File
@@ -1,7 +1,8 @@
// For more information on writing tests, see
// https://scalameta.org/munit/docs/getting-started.html
import compiler.Tokenizer.*
import compiler.Tokenizer._
import compiler.Tokenizer.TokenType._
import scala.util.{Try, Failure, Success}
@@ -25,11 +26,11 @@ class TokenizerTestWithoutLocation extends munit.FunSuite {
test("should work with identifiers") {
val code = "while true do print_int BigInt"
val expected = List(
IdentifierT("while", L),
IdentifierT("true", L),
IdentifierT("do", L),
IdentifierT("print_int", L),
IdentifierT("BigInt", L),
Token("while", Identifier, L),
Token("true", Identifier, L),
Token("do", Identifier, L),
Token("print_int", Identifier, L),
Token("BigInt", Identifier, L),
)
test_tokenizer(code, expected)
}
@@ -38,11 +39,11 @@ class TokenizerTestWithoutLocation extends munit.FunSuite {
val code = "41 + 3 == 44"
val expected =
List(
IntLiteralT("41", L),
OperatorT("+", L),
IntLiteralT("3", L),
OperatorT("==", L),
IntLiteralT("44", L)
Token("41", IntLiteral, L),
Token("+", Operator, L),
Token("3", IntLiteral, L),
Token("==", Operator, L),
Token("44", IntLiteral, L)
)
test_tokenizer(code, expected)
}
@@ -50,7 +51,13 @@ class TokenizerTestWithoutLocation extends munit.FunSuite {
test("should work with operators and int literals") {
val code = "1531 + 1241 == 2772"
val expected =
List(IntLiteralT("1531", L), OperatorT("+", L), IntLiteralT("1241", L), OperatorT("==", L), IntLiteralT("2772", L))
List(
Token("1531", IntLiteral, L),
Token("+", Operator, L),
Token("1241", IntLiteral, L),
Token("==", Operator, L),
Token("2772", IntLiteral, L)
)
test_tokenizer(code, expected)
}
@@ -59,11 +66,11 @@ class TokenizerTestWithoutLocation extends munit.FunSuite {
val code2 = "4/2==2 // math"
val code3 = "# this is a whole line comment"
val expected = List(
IntLiteralT("4", L),
OperatorT("/", L),
IntLiteralT("2", L),
OperatorT("==", L),
IntLiteralT("2", L),
Token("4", IntLiteral, L),
Token("/", Operator, L),
Token("2", IntLiteral, L),
Token("==", Operator, L),
Token("2", IntLiteral, L),
)
test_tokenizer(code1, expected)
test_tokenizer(code2, expected)
@@ -80,11 +87,11 @@ class TokenizerTestWithoutLocation extends munit.FunSuite {
test("should work with function parenthesis") {
val code = "print_int(n);"
val expected = List(
IdentifierT("print_int", L),
PunctuationT("(", L),
IdentifierT("n", L),
PunctuationT(")", L),
PunctuationT(";", L),
Token("print_int", Identifier, L),
Token("(", Punctuation, L),
Token("n", Identifier, L),
Token(")", Punctuation, L),
Token(";", Punctuation, L),
)
test_tokenizer(code, expected)
}
@@ -92,17 +99,17 @@ class TokenizerTestWithoutLocation extends munit.FunSuite {
test("should work with functions with types") {
val code = "test(i: Int,j: Float);"
val expected = List(
IdentifierT("test", L),
PunctuationT("(", L),
IdentifierT("i", L),
PunctuationT(":", L),
IdentifierT("Int", L),
PunctuationT(",", L),
IdentifierT("j", L),
PunctuationT(":", L),
IdentifierT("Float", L),
PunctuationT(")", L),
PunctuationT(";", L),
Token("test", Identifier, L),
Token("(", Punctuation, L),
Token("i", Identifier, L),
Token(":", Punctuation, L),
Token("Int", Identifier, L),
Token(",", Punctuation, L),
Token("j", Identifier, L),
Token(":", Punctuation, L),
Token("Float", Identifier, L),
Token(")", Punctuation, L),
Token(";", Punctuation, L),
)
test_tokenizer(code, expected)
}
@@ -110,18 +117,18 @@ class TokenizerTestWithoutLocation extends munit.FunSuite {
test("should work with more complicated control flow") {
val code = "if (i>0) { it(); }"
val expected = List(
IdentifierT("if", L),
PunctuationT("(", L),
IdentifierT("i", L),
OperatorT(">", L),
IntLiteralT("0", L),
PunctuationT(")", L),
PunctuationT("{", L),
IdentifierT("it", L),
PunctuationT("(", L),
PunctuationT(")", L),
PunctuationT(";", L),
PunctuationT("}", L),
Token("if", Identifier, L),
Token("(", Punctuation, L),
Token("i", Identifier, L),
Token(">", Operator, L),
Token("0", IntLiteral, L),
Token(")", Punctuation, L),
Token("{", Punctuation, L),
Token("it", Identifier, L),
Token("(", Punctuation, L),
Token(")", Punctuation, L),
Token(";", Punctuation, L),
Token("}", Punctuation, L),
)
test_tokenizer(code, expected)
}
@@ -139,7 +146,7 @@ class TokenizerTestWithLocation extends munit.FunSuite {
test("should fail with incorrect positions") {
val code = "1453"
val location = TokenLocation(Position(2, 1), Position(4,10))
val expected = List(IntLiteralT(code, location))
val expected = List(Token(code, IntLiteral, location))
val result = Tokenizer.tokenize(code)
result match {
case Success(result) => assertNotEquals(result, expected)
@@ -150,16 +157,16 @@ class TokenizerTestWithLocation extends munit.FunSuite {
test("should work with int literals") {
val code = "1453"
val location = TokenLocation(Position(1, 1), Position(1,5))
val expected = List(IntLiteralT(code, location))
val expected = List(Token(code, IntLiteral, location))
test_tokenizer(code, expected)
}
test("should work with a newline") {
val code = "if 3\nwhile"
val expected = List(
IdentifierT("if", TokenLocation(Position(1,1), Position(1,3))),
IntLiteralT("3", TokenLocation(Position(1,5), Position(1,6))),
IdentifierT("while", TokenLocation(Position(2,1), Position(2,6)))
Token("if", Identifier, TokenLocation(Position(1,1), Position(1,3))),
Token("3", IntLiteral, TokenLocation(Position(1,5), Position(1,6))),
Token("while", Identifier, TokenLocation(Position(2,1), Position(2,6)))
)
test_tokenizer(code, expected)
}
@@ -167,10 +174,10 @@ class TokenizerTestWithLocation extends munit.FunSuite {
test("should work with multiple newline") {
val code = "if 3\nwhile\n\nprint_int"
val expected = List(
IdentifierT("if", TokenLocation(Position(1,1), Position(1,3))),
IntLiteralT("3", TokenLocation(Position(1,5), Position(1,6))),
IdentifierT("while", TokenLocation(Position(2,1), Position(2,6))),
IdentifierT("print_int", TokenLocation(Position(4,1), Position(4,10)))
Token("if", Identifier, TokenLocation(Position(1,1), Position(1,3))),
Token("3", IntLiteral, TokenLocation(Position(1,5), Position(1,6))),
Token("while", Identifier, TokenLocation(Position(2,1), Position(2,6))),
Token("print_int", Identifier, TokenLocation(Position(4,1), Position(4,10)))
)
test_tokenizer(code, expected)
}
@@ -178,17 +185,17 @@ class TokenizerTestWithLocation extends munit.FunSuite {
test("should work with multiple tokens") {
val code = "1 + 2 % 4 / 3 * 3 <= 100"
val expected = List(
IntLiteralT("1", TokenLocation(Position(1,1), Position(1,2))),
OperatorT("+", TokenLocation(Position(1,3), Position(1,4))),
IntLiteralT("2", TokenLocation(Position(1,5), Position(1,6))),
OperatorT("%", TokenLocation(Position(1,7), Position(1,8))),
IntLiteralT("4", TokenLocation(Position(1,9), Position(1,10))),
OperatorT("/", TokenLocation(Position(1,11), Position(1,12))),
IntLiteralT("3", TokenLocation(Position(1,13), Position(1,14))),
OperatorT("*", TokenLocation(Position(1,15), Position(1,16))),
IntLiteralT("3", TokenLocation(Position(1,17), Position(1,18))),
OperatorT("<=", TokenLocation(Position(1,19), Position(1,21))),
IntLiteralT("100", TokenLocation(Position(1,22), Position(1,25)))
Token("1", IntLiteral, TokenLocation(Position(1,1), Position(1,2))),
Token("+", Operator, TokenLocation(Position(1,3), Position(1,4))),
Token("2", IntLiteral, TokenLocation(Position(1,5), Position(1,6))),
Token("%", Operator, TokenLocation(Position(1,7), Position(1,8))),
Token("4", IntLiteral, TokenLocation(Position(1,9), Position(1,10))),
Token("/", Operator, TokenLocation(Position(1,11), Position(1,12))),
Token("3", IntLiteral, TokenLocation(Position(1,13), Position(1,14))),
Token("*", Operator, TokenLocation(Position(1,15), Position(1,16))),
Token("3", IntLiteral, TokenLocation(Position(1,17), Position(1,18))),
Token("<=", Operator, TokenLocation(Position(1,19), Position(1,21))),
Token("100", IntLiteral, TokenLocation(Position(1,22), Position(1,25)))
)
test_tokenizer(code, expected)
}