add: tokenizer for content

2026-08-23 22:55:16 +05:30
parent 4b2ecb367d
commit 3bc9e06673
4 changed files with 90 additions and 6 deletions
@@ -6,3 +6,6 @@ edition = "2024"
 [dependencies]
 log = "0.4.34"
 plume-log = "0.3.0"
+proc-macro2 = "1.0.107"
+regex = "1.13.1"
+token-string = "0.8.4"