7 lines
288 B
Python
7 lines
288 B
Python
from smnp.token.tools import regexPatternTokenizer
|
|
from smnp.token.type import TokenType
|
|
|
|
def identifierTokenizer(input, current, line):
|
|
# TODO: Disallow to create identifiers beggining from a number
|
|
return regexPatternTokenizer(TokenType.IDENTIFIER, r'\w')(input, current, line)
|