add: tokenizer for content

src/main.rs

@@ -1,11 +1,13 @@
-#![allow(unused_imports)]
+#![allow(unused)]
 #![allow(dead_code)]
 
 use std::env;
 use std::fs::File;
+use std::fs;
 use std::io::{BufRead, BufReader};
 use log::{debug, error, info, trace, warn};
 use plume_log::{PlumeBuilder, success};
+use regex::Regex;
 
 struct SymbolRow {
     sym: String,
@@ -30,12 +32,14 @@ fn main() -> std::io::Result<()>{
 	std::process::exit(-1);
     }
     info!("File: {}", file_path);
-    let file = File::open(&args[1])?;
-    let reader = BufReader::new(file);
+    let content = fs::read_to_string(file_path)
+	.expect("Should've read the file");
 
-    for line in reader.lines() {
-	println!("+ {}", line?);
+    let re = Regex::new(r"[ \n\t]+").unwrap();
+    let tokens: Vec<&str> = re.split(&content).filter(|s| !s.is_empty()).collect();
+    for token in tokens {
+	print!("{token} ",);
     }
-
+    
     Ok(())
 }