Refactor tokenizer

2019-07-06 22:09:01 +02:00
parent fbb3f79731
commit 756f4544e4
18 changed files with 59 additions and 156 deletions
--- a/smnp/token/tokenizers/comment.py
+++ b/smnp/token/tokenizers/comment.py
@@ -1,7 +1,8 @@
-from smnp.token.type import TokenType
 from smnp.token.model import Token
+from smnp.token.type import TokenType

-def tokenizeComment(input, current, line):
+
+def commentTokenizer(input, current, line):
    if input[current] == '#':
        consumedChars = 0
        value = ''