From 3bc9e06673c68fbd71c7409276dd4ad627363aee Mon Sep 17 00:00:00 2001
From: veradec <glitchmill.dyj3d@simplelogin.co>
Date: Sun, 23 Aug 2026 22:55:16 +0530
Subject: [PATCH] add: tokenizer for content

---
 Cargo.lock        | 68 +++++++++++++++++++++++++++++++++++++++++++++++
 Cargo.toml        |  3 +++
 examples/print.sr |  9 +++++++
 src/main.rs       | 16 ++++++-----
 4 files changed, 90 insertions(+), 6 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 0b5b1b1..3e84207 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2,6 +2,15 @@
 # It is not intended for manual editing.
 version = 4
 
+[[package]]
+name = "aho-corasick"
+version = "1.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "hermit-abi"
 version = "0.5.2"
@@ -37,6 +46,12 @@ version = "0.4.34"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6"
 
+[[package]]
+name = "memchr"
+version = "2.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
+
 [[package]]
 name = "owo-colors"
 version = "4.3.0"
@@ -57,12 +72,53 @@ dependencies = [
  "owo-colors",
 ]
 
+[[package]]
+name = "proc-macro2"
+version = "1.0.107"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "regex"
+version = "1.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-automata",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-automata"
+version = "0.4.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-syntax"
+version = "0.8.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
+
 [[package]]
 name = "shiro"
 version = "0.1.0"
 dependencies = [
  "log",
  "plume-log",
+ "proc-macro2",
+ "regex",
+ "token-string",
 ]
 
 [[package]]
@@ -84,6 +140,18 @@ dependencies = [
  "is_ci",
 ]
 
+[[package]]
+name = "token-string"
+version = "0.8.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ed962f478bffb24bef4387092d43a47b5bee907377e93e9aa11465495bef4067"
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
+
 [[package]]
 name = "windows-link"
 version = "0.2.1"
diff --git a/Cargo.toml b/Cargo.toml
index ac42604..d0b9c4c 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -6,3 +6,6 @@ edition = "2024"
 [dependencies]
 log = "0.4.34"
 plume-log = "0.3.0"
+proc-macro2 = "1.0.107"
+regex = "1.13.1"
+token-string = "0.8.4"
diff --git a/examples/print.sr b/examples/print.sr
index c3b30ec..b9e8210 100644
--- a/examples/print.sr
+++ b/examples/print.sr
@@ -10,4 +10,13 @@ main() {
      	   x++;
      }
      print("x is % now", x);
+
+     string file_path = "../examples/print.sr";
+     fs.read(file_path);
+
+
+     uint8[] x = [1,2,3];
+     x.push(4);
+     x.push(1);
+     x.push(2);
 }
\ No newline at end of file
diff --git a/src/main.rs b/src/main.rs
index 24a1317..2d379db 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -1,11 +1,13 @@
-#![allow(unused_imports)]
+#![allow(unused)]
 #![allow(dead_code)]
 
 use std::env;
 use std::fs::File;
+use std::fs;
 use std::io::{BufRead, BufReader};
 use log::{debug, error, info, trace, warn};
 use plume_log::{PlumeBuilder, success};
+use regex::Regex;
 
 struct SymbolRow {
     sym: String,
@@ -30,12 +32,14 @@ fn main() -> std::io::Result<()>{
 	std::process::exit(-1);
     }
     info!("File: {}", file_path);
-    let file = File::open(&args[1])?;
-    let reader = BufReader::new(file);
+    let content = fs::read_to_string(file_path)
+	.expect("Should've read the file");
 
-    for line in reader.lines() {
-	println!("+ {}", line?);
+    let re = Regex::new(r"[ \n\t]+").unwrap();
+    let tokens: Vec<&str> = re.split(&content).filter(|s| !s.is_empty()).collect();
+    for token in tokens {
+	print!("{token} ",);
     }
-
+    
     Ok(())
 }