Refactor tokenizer: remove colon, add colon as duration separator to note and create TokenType.TYPE

smnp/token/tools.py

@@ -1,6 +1,8 @@
 import re
+
 from smnp.token.model import Token
 
+
 def tokenizeChar(type, char, input, current, line):
     if input[current] == char:
         return (1, Token(type, input[current], (line, current)))
@@ -15,6 +17,13 @@ def tokenizeRegexPattern(type, pattern, input, current, line):
         consumedChars += 1            
     return (consumedChars, Token(type, value, (line, current)) if consumedChars > 0 else None)
 
+def tokenizeKeywords(type, input, current, line, *keywords):
+    for keyword in keywords:
+        result = tokenizeKeyword(type, keyword, input, current, line)
+        if result[0] > 0:
+            return result
+    return (0, None)
+
 def tokenizeKeyword(type, keyword, input, current, line):       
     if len(input) >= current+len(keyword) and input[current:current+len(keyword)] == keyword:
         return (len(keyword), Token(type, keyword, (line, current)))