Refactor tokenizer

2019-07-03 01:55:08 +02:00
parent 8313d2dcfd
commit f826516d8f
41 changed files with 589 additions and 296 deletions
--- a/smnp/token/tokenizers/whitespace.py
+++ b/smnp/token/tokenizers/whitespace.py
@@ -0,0 +1,4 @@
+from smnp.token.tools import tokenizeRegexPattern
+
+def tokenizeWhitespaces(input, current, line):    
+    return tokenizeRegexPattern(None, r'\s', input, current, line)