Refactor tokenizer

2019-07-06 22:09:01 +02:00
parent fbb3f79731
commit 756f4544e4
18 changed files with 59 additions and 156 deletions
--- a/smnp/token/tokenizers/identifier.py
+++ b/smnp/token/tokenizers/identifier.py
@@ -1,6 +1,6 @@
 from smnp.token.tools import regexPatternTokenizer
 from smnp.token.type import TokenType

-def tokenizeIdentifier(input, current, line):
+def identifierTokenizer(input, current, line):
    # TODO: Disallow to create identifiers beggining from a number
    return regexPatternTokenizer(TokenType.IDENTIFIER, r'\w')(input, current, line)