seems like a working tokenizer

Dockerfile

@@ -16,4 +16,4 @@ COPY --from=builder /app/compiler/target/release/compiler .
 
 EXPOSE 3000
 
-CMD ["./compiler"]
+CMD ["./compiler", "serve", "'<placeholder>'"]

compiler/Cargo.lock

@@ -2,6 +2,15 @@
 # It is not intended for manual editing.
 version = 4
 
+[[package]]
+name = "aho-corasick"
+version = "1.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "base64"
 version = "0.22.1"
@@ -14,6 +23,7 @@ version = "0.1.0"
 dependencies = [
  "base64",
  "json",
+ "regex",
 ]
 
 [[package]]
@@ -21,3 +31,38 @@ name = "json"
 version = "0.12.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "078e285eafdfb6c4b434e0d31e8cfcb5115b651496faca5749b88fafd4f23bfd"
+
+[[package]]
+name = "memchr"
+version = "2.7.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273"
+
+[[package]]
+name = "regex"
+version = "1.12.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-automata",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-automata"
+version = "0.4.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-syntax"
+version = "0.8.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58"

compiler/Cargo.toml

@@ -6,3 +6,4 @@ edition = "2024"
 [dependencies]
 base64 = "0.22.1"
 json = "0.12.4"
+regex = "1.12.2"

compiler/src/compiler/lexer.rs

@@ -0,0 +1 @@
+pub fn lexer() {}

compiler/src/compiler/mod.rs

@@ -1,4 +1,8 @@
+pub mod lexer;
+pub mod tokenizer;
+
 pub fn compile(code: &str) -> &str {
+    tokenizer::tokenizer(code);
     println!("Code: {}", &code);
     return code;
 }

compiler/src/compiler/tokenizer.rs

@@ -0,0 +1,49 @@
+use regex::Regex;
+
+pub fn tokenizer(code: &str) {
+    // Damn, regex...
+    println!("Doing regex things for {}", code);
+    let re = Regex::new(
+        r#"(?x)
+            (((\#|\/\/).*)|              # Comments
+            ([+\-\/*%(),;:{}])|         # Single character operators, e.g. +,-,*, ...
+            ([<>=!]=?)|                 # (Potentially) Multicharacter operators, e.g. <,!,=, ... (<=, !=, ==)
+            ([0-9]+)|                   # Numbers
+            ([a-zA-Z_][a-zA-Z_0-9]*))   # Identifiers
+        "#,
+    )
+    .unwrap();
+
+    let tokens: Vec<&str> = re
+        .find_iter(code)
+        .map(|mat| {
+            let token = mat.as_str();
+            println!("{token}");
+            token
+        })
+        .collect();
+    println!("{tokens:?}");
+    // return tokens;
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn single_integer_literals() {
+        let code = "143";
+        let result = tokenizer(code);
+
+        // assert_eq!(result[0], "143");
+    }
+
+    #[test]
+    fn multiple_integer_literals() {
+        let code = "532 125390";
+        let result = tokenizer(code);
+
+        // assert_eq!(result[0], "532");
+        // assert_eq!(result[1], "125390");
+    }
+}

compiler/src/main.rs

@@ -1,4 +1,5 @@
 use base64::prelude::*;
+use std::env;
 use std::{
     io::{BufReader, prelude::*},
     net::{TcpListener, TcpStream},
@@ -10,6 +11,19 @@ const PORT: &str = "3000";
 const HOST: &str = "0.0.0.0";
 
 fn main() {
+    let args: Vec<_> = env::args().collect();
+    if args.len() <= 2 {
+        println!("Too few arguments, try either `compile` or `serve`");
+        println!("If using serve, just add something random as 2nd argument :D");
+        return;
+    }
+
+    match args[1].as_str() {
+        "compile" => {
+            let executable = compiler::compile(args[2].as_str());
+            println!("Executable: {}", executable);
+        }
+        "serve" => {
             let address = format!("{}:{}", HOST, PORT);
             let listener = TcpListener::bind(&address).unwrap();
 
@@ -21,6 +35,11 @@ fn main() {
                 handle_connection(stream);
             }
         }
+        _ => {
+            println!("Incorrect usage!");
+        }
+    }
+}
 
 fn handle_connection(mut stream: TcpStream) {
     let buf_reader = BufReader::new(&stream);