finished tokenizer and tests

.scalafmt.conf

@@ -0,0 +1,2 @@
+version = 3.10.5
+runner.dialect = scala3

compiler/src/main/scala/Main.scala

@@ -4,7 +4,7 @@ import java.io.{BufferedReader, InputStreamReader, PrintWriter}
 import java.net.{ServerSocket, Socket};
 import java.util.stream.Stream
 
-import compiler.Tokenizer.tokenizer
+import compiler.Tokenizer._
 
 
 @main def hello(): Unit =
@@ -45,4 +45,4 @@ def run_server(): Unit =
     out.println(response);
 
 def compile(source: String): String =
-  return tokenizer(source).toString()
+  return Tokenizer.tokenize(source).toString()

compiler/src/main/scala/Tokenizer.scala

@@ -1,44 +1,110 @@
-package compiler.Tokenizer;
+package compiler.Tokenizer
 
-import scala.util.matching.Regex.MatchIterator;
+import scala.util.matching.Regex.MatchIterator
+import scala.util.{Try, Failure, Success}
 
-class TokenLocation(column: Int, line: Int):
+
+val TabLength = 4
+
+
+class Position(var line: Int, var column: Int):
   override def toString: String = s"($line,$column)"
+  override def equals(other: Any): Boolean = other match {
+    case that: Position => (that.line == this.line) && (that.column == this.column)
+    case _ => false
+  }
+
+class TokenLocation(val start: Position, val end: Position):
+  override def toString: String = s"TokenLoc(S:$start, E:$end)"
+  override def equals(other: Any): Boolean = other match {
+    case that: TokenLocation => (that.start == this.start) && (that.end == this.end)
+    case that: TokenLocationDebug => true
+    case _ => false
+  }
 
 class TokenLocationDebug():
-  override def toString: String =
-    s"(debug-loc)"
+  override def toString: String = s"(debug-loc)"
+  override def equals(other: Any): Boolean = other match {
+    case that: TokenLocation => true
+    case that: TokenLocationDebug => true
+    case _ => false
+  }
 
-class Token(
-    value: String,
-    location: TokenLocation | TokenLocationDebug = TokenLocationDebug()
-):
-  override def toString: String =
-    s"Token($value, $location)"
+sealed trait Token(val value: String, val location: TokenLocation | TokenLocationDebug = TokenLocationDebug()):
+  override def toString: String = s"${this.getClass.getName.stripPrefix("compiler.Tokenizer.")}($value, $location)"
+  override def equals(other: Any): Boolean = other match {
+    case that: Token => {
+      that.location match {
+        case x: TokenLocation => (that.value == this.value) && (that.location == this.location)
+        case x: TokenLocationDebug => true
+      }
+    }
+    case _ => false
+  }
+  def isSkippable() =
+    this match {
+      case that: WhitespaceT => true
+      case that: CommentT => true
+      case _ => false
+    }
 
 object Token:
-  def tokenFromString(str: String): Token =
-    return Token("test")
+  def tokenFromString(str: String, currentPos: Position): Option[Token] =
 
-  def isIntLiteral(str: String): Boolean =
-    return true
 
-class CurrentPosition(line: Int, column: Int);
+    if Token.isWhitespace(str) then 
+      val startWhitespace = Position(currentPos.line, currentPos.column)
+      str.foreach(_ match {
+        case ' ' => currentPos.column += 1
+        case '\n' => currentPos.line += 1; currentPos.column = 1
+        case '\t' => currentPos.column += TabLength
+      })
 
-def tokenizer(source: String): List[Token] =
+      val endWhitespace = Position(currentPos.line, currentPos.column)
+      val tokenLoc = TokenLocation(startWhitespace, endWhitespace)
+      return Some(WhitespaceT(str, tokenLoc))
+
+    // After whitespace is calculated, move currentPos to the end of the current token
+    // i.e. current column + token length
+    val start = Position(currentPos.line, currentPos.column)
+    currentPos.column += str.length()
+    val end = Position(currentPos.line, currentPos.column)
+    val tokenLoc = TokenLocation(start, end)
+
+    if Token.isComment(str) then return Some(CommentT(str, tokenLoc))
+    if Token.isIntLiteral(str) then return Some(IntLiteralT(str, tokenLoc))
+    if Token.isOperator(str) then return Some(OperatorT(str, tokenLoc))
+    if Token.isIdentifier(str) then return Some(IdentifierT(str, tokenLoc))
+    if Token.isPunctuation(str) then return Some(PunctuationT(str, tokenLoc))
+    else return None
+
+
+  def isWhitespace(str: String): Boolean = return raw"(\s+)".r.matches(str)
+  def isComment(str: String): Boolean = return str.startsWith("#") || str.startsWith("//")
+  def isIntLiteral(str: String): Boolean = return str.forall(Character.isDigit)
+  def isOperator(str: String): Boolean = return raw"([+\-\/*%])|([<>=!]=?)".r.matches(str)
+  def isPunctuation(str: String): Boolean = raw"[(),;:{}]".r.matches(str)
+  def isIdentifier(str: String): Boolean = raw"[a-zA-Z_][a-zA-Z_0-9]*".r.matches(str)
+
+class WhitespaceT(value: String, location: TokenLocation | TokenLocationDebug) extends Token(value, location)
+class CommentT(value: String, location: TokenLocation | TokenLocationDebug) extends Token(value, location)
+class IntLiteralT(value: String, location: TokenLocation | TokenLocationDebug) extends Token(value, location)
+class OperatorT(value: String, location: TokenLocation | TokenLocationDebug) extends Token(value, location)
+class PunctuationT(value: String, location: TokenLocation | TokenLocationDebug) extends Token(value, location)
+class IdentifierT(value: String, location: TokenLocation | TokenLocationDebug) extends Token(value, location)
+class OtherT(value: String, location: TokenLocation | TokenLocationDebug) extends Token(value, location)
+
+
+object Tokenizer:
+  def tokenize(source: String): Try[List[Token]] =
     val tokenRegex =
-    raw"(((\#|\/\/).*)|([+\-\/*%(),;:{}])|([<>=!]=?)|([0-9]+)|([a-zA-Z_][a-zA-Z_0-9]*))".r
+      raw"(\s+)|(((?:\#|\/\/).*$$)|([+\-\/*%])|([(),;:{}])|([<>=!]=?)|([0-9]+)|([a-zA-Z_][a-zA-Z_0-9]*))".r
     val matches = tokenRegex.findAllIn(source)
 
-  var currentPos = CurrentPosition(0, 0)
-  val tokens = for (token <- matches) yield tokenFromMatch(token, currentPos)
+    var currentPos = Position(1, 1)
+    val tokens = matches.map(token => Token.tokenFromString(token, currentPos)).toList
 
-  return tokens.toList
-
-def tokenFromMatch(tokenStr: String, curPos: CurrentPosition): Token =
-  val location = TokenLocationDebug()
-  val value = tokenStr
-
-  val token = Token(value, location)
-  println("Token: " + token)
-  return token
+    if tokens.exists(_.isEmpty) 
+      then Failure(new Exception("Incorrect tokens found"))
+    else
+      Success(tokens.flatten.filter(!_.isSkippable()))

compiler/src/test/scala/MySuite.scala

@@ -2,34 +2,194 @@
 // https://scalameta.org/munit/docs/getting-started.html
 
 import compiler.Tokenizer.*
+import scala.util.{Try, Failure, Success}
+
+
+class TokenizerTestWithoutLocation extends munit.FunSuite {
+  val L = TokenLocationDebug()
+
+  def test_tokenizer(code: String, expected: List[Token]): Unit =
+    val result = Tokenizer.tokenize(code)
+    result match {
+      case Success(result) => assertEquals(result, expected)
+      case Failure(result) => fail("Tokenizing failed: " + result)
+    }
+
 
-class TokenizerTest extends munit.FunSuite {
   test("test test succeeds") {
     val obtained = 42
     val expected = 42
     assertEquals(obtained, expected)
   }
 
-  test("should work with int literals") {
-    val code = "1453"
-    val expected = List(Token(code))
-    val result = tokenizer(code)
-    assertEquals(result.toString(), expected.toString())
+  test("should work with identifiers") {
+    val code = "while true do print_int BigInt"
+    val expected = List(
+      IdentifierT("while", L),
+      IdentifierT("true", L),
+      IdentifierT("do", L),
+      IdentifierT("print_int", L),
+      IdentifierT("BigInt", L),
+    )
+    test_tokenizer(code, expected)
   }
 
-  test("should work with simple operatorn and int literals") {
+  test("should work with simple operators and int literals") {
     val code = "41 + 3 == 44"
     val expected =
-      List(Token("41"), Token("+"), Token("3"), Token("=="), Token("44"))
-    val result = tokenizer(code)
-    assertEquals(result.toString(), expected.toString())
+      List(
+        IntLiteralT("41", L),
+        OperatorT("+", L),
+        IntLiteralT("3", L),
+        OperatorT("==", L),
+        IntLiteralT("44", L)
+      )
+    test_tokenizer(code, expected)
   }
 
   test("should work with operators and int literals") {
     val code = "1531 + 1241 == 2772"
     val expected =
-      List(Token("1531"), Token("+"), Token("1241"), Token("=="), Token("2772"))
-    val result = tokenizer(code)
-    assertEquals(result.toString(), expected.toString())
+      List(IntLiteralT("1531", L), OperatorT("+", L), IntLiteralT("1241", L), OperatorT("==", L), IntLiteralT("2772", L))
+    test_tokenizer(code, expected)
+  }
+
+  test("should work with comments") {
+    val code1 = "4/2==2 #math"
+    val code2 = "4/2==2 // math"
+    val code3 = "# this is a whole line comment"
+    val expected = List(
+      IntLiteralT("4", L),
+      OperatorT("/", L),
+      IntLiteralT("2", L),
+      OperatorT("==", L),
+      IntLiteralT("2", L),
+    )
+    test_tokenizer(code1, expected)
+    test_tokenizer(code2, expected)
+    test_tokenizer(code3, Nil)
+  }
+
+  test("should work with whole-line comments") {
+    val code = "# this is a whole line comment"
+    val expected = Nil
+    test_tokenizer(code, expected)
+  }
+
+
+  test("should work with function parenthesis") {
+    val code = "print_int(n);"
+    val expected = List(
+      IdentifierT("print_int", L),
+      PunctuationT("(", L),
+      IdentifierT("n", L),
+      PunctuationT(")", L),
+      PunctuationT(";", L),
+    )
+    test_tokenizer(code, expected)
+  }
+
+  test("should work with functions with types") {
+    val code = "test(i: Int,j: Float);"
+    val expected = List(
+      IdentifierT("test", L),
+      PunctuationT("(", L),
+      IdentifierT("i", L),
+      PunctuationT(":", L),
+      IdentifierT("Int", L),
+      PunctuationT(",", L),
+      IdentifierT("j", L),
+      PunctuationT(":", L),
+      IdentifierT("Float", L),
+      PunctuationT(")", L),
+      PunctuationT(";", L),
+    )
+    test_tokenizer(code, expected)
+  }
+
+  test("should work with more complicated control flow") {
+    val code = "if (i>0) { it(); }"
+    val expected = List(
+      IdentifierT("if", L),
+      PunctuationT("(", L),
+      IdentifierT("i", L),
+      OperatorT(">", L),
+      IntLiteralT("0", L),
+      PunctuationT(")", L),
+      PunctuationT("{", L),
+      IdentifierT("it", L),
+      PunctuationT("(", L),
+      PunctuationT(")", L),
+      PunctuationT(";", L),
+      PunctuationT("}", L),
+    )
+    test_tokenizer(code, expected)
+  }
+}
+
+
+class TokenizerTestWithLocation extends munit.FunSuite {
+  def test_tokenizer(code: String, expected: List[Token]): Unit =
+    val result = Tokenizer.tokenize(code)
+    result match {
+      case Success(result) => assertEquals(result, expected)
+      case Failure(result) => fail("Tokenizing failed: " + result)
+    }
+
+  test("should fail with incorrect positions") {
+    val code = "1453"
+    val location = TokenLocation(Position(2, 1), Position(4,10))
+    val expected = List(IntLiteralT(code, location))
+    val result = Tokenizer.tokenize(code)
+    result match {
+      case Success(result) => assertNotEquals(result, expected)
+      case Failure(result) => fail("Tokenizing failed: " + result)
+    }
+  }
+
+  test("should work with int literals") {
+    val code = "1453"
+    val location = TokenLocation(Position(1, 1), Position(1,5))
+    val expected = List(IntLiteralT(code, location))
+    test_tokenizer(code, expected)
+  }
+
+  test("should work with a newline") {
+    val code = "if  3\nwhile"
+    val expected = List(
+      IdentifierT("if", TokenLocation(Position(1,1), Position(1,3))),
+      IntLiteralT("3", TokenLocation(Position(1,5), Position(1,6))),
+      IdentifierT("while", TokenLocation(Position(2,1), Position(2,6)))
+    )
+    test_tokenizer(code, expected)
+  }
+
+  test("should work with multiple newline") {
+    val code = "if  3\nwhile\n\nprint_int"
+    val expected = List(
+      IdentifierT("if", TokenLocation(Position(1,1), Position(1,3))),
+      IntLiteralT("3", TokenLocation(Position(1,5), Position(1,6))),
+      IdentifierT("while", TokenLocation(Position(2,1), Position(2,6))),
+      IdentifierT("print_int", TokenLocation(Position(4,1), Position(4,10)))
+    )
+    test_tokenizer(code, expected)
+  }
+
+  test("should work with multiple tokens") {
+    val code = "1 + 2 % 4 / 3 * 3 <= 100"
+    val expected = List(
+      IntLiteralT("1", TokenLocation(Position(1,1), Position(1,2))),
+      OperatorT("+", TokenLocation(Position(1,3), Position(1,4))),
+      IntLiteralT("2", TokenLocation(Position(1,5), Position(1,6))),
+      OperatorT("%", TokenLocation(Position(1,7), Position(1,8))),
+      IntLiteralT("4", TokenLocation(Position(1,9), Position(1,10))),
+      OperatorT("/", TokenLocation(Position(1,11), Position(1,12))),
+      IntLiteralT("3", TokenLocation(Position(1,13), Position(1,14))),
+      OperatorT("*", TokenLocation(Position(1,15), Position(1,16))),
+      IntLiteralT("3", TokenLocation(Position(1,17), Position(1,18))),
+      OperatorT("<=", TokenLocation(Position(1,19), Position(1,21))),
+      IntLiteralT("100", TokenLocation(Position(1,22), Position(1,25)))
+    )
+    test_tokenizer(code, expected)
   }
 }

query_server.sh

@@ -0,0 +1 @@
+echo '{"command": "compile", "code": "1+1 == 2"}' | netcat localhost 3000