diff --git a/smnp/token/tokenizer.py b/smnp/token/tokenizer.py index a269b1d..cc96c14 100644 --- a/smnp/token/tokenizer.py +++ b/smnp/token/tokenizer.py @@ -3,7 +3,6 @@ from smnp.token.model import TokenList from smnp.token.tokenizers.assign import tokenizeAssign from smnp.token.tokenizers.asterisk import tokenizeAsterisk from smnp.token.tokenizers.bracket import tokenizeOpenBracket, tokenizeCloseBracket -from smnp.token.tokenizers.colon import tokenizeColon from smnp.token.tokenizers.comma import tokenizeComma from smnp.token.tokenizers.comment import tokenizeComment from smnp.token.tokenizers.dot import tokenizeDot @@ -17,6 +16,7 @@ from smnp.token.tokenizers.percent import tokenizePercent from smnp.token.tokenizers.ret import tokenizeReturn from smnp.token.tokenizers.square import tokenizeOpenSquare, tokenizeCloseSquare from smnp.token.tokenizers.string import tokenizeString +from smnp.token.tokenizers.type import tokenizeType from smnp.token.tokenizers.whitespace import tokenizeWhitespaces from smnp.token.type import TokenType @@ -25,7 +25,8 @@ tokenizers = ( tokenizeCloseParen, tokenizeOpenSquare, tokenizeCloseSquare, - tokenizeAsterisk, + tokenizeAsterisk, + tokenizeType, tokenizeString, tokenizeFunction, tokenizeReturn, @@ -36,7 +37,6 @@ tokenizers = ( tokenizeOpenBracket, tokenizeCloseBracket, tokenizeAssign, - tokenizeColon, tokenizePercent, tokenizeMinus, tokenizeDot, diff --git a/smnp/token/tokenizers/colon.py b/smnp/token/tokenizers/colon.py deleted file mode 100644 index 05a6571..0000000 --- a/smnp/token/tokenizers/colon.py +++ /dev/null @@ -1,5 +0,0 @@ -from smnp.token.tools import tokenizeChar -from smnp.token.type import TokenType - -def tokenizeColon(input, current, line): - return tokenizeChar(TokenType.COLON, ':', input, current, line) diff --git a/smnp/token/tokenizers/note.py b/smnp/token/tokenizers/note.py index 3d19976..b578b44 100644 --- a/smnp/token/tokenizers/note.py +++ b/smnp/token/tokenizers/note.py @@ -25,7 +25,7 @@ def tokenizeNote(input, current, line): octave = input[current+consumedChars] consumedChars += 1 - if current+consumedChars < len(input) and input[current+consumedChars] == '^': + if current+consumedChars < len(input) and input[current+consumedChars] == ':': duration = '' consumedChars += 1 while current+consumedChars < len(input) and re.match(r'\d', input[current+consumedChars]): diff --git a/smnp/token/tokenizers/type.py b/smnp/token/tokenizers/type.py new file mode 100644 index 0000000..a42f009 --- /dev/null +++ b/smnp/token/tokenizers/type.py @@ -0,0 +1,8 @@ +from smnp.token.tools import tokenizeKeywords +from smnp.token.type import TokenType +from smnp.type.model import Type + + +def tokenizeType(input, current, line): + types = [ type.name.lower() for type in Type ] + return tokenizeKeywords(TokenType.TYPE, input, current, line, *types) \ No newline at end of file diff --git a/smnp/token/tools.py b/smnp/token/tools.py index 9f4e902..e0ebc09 100644 --- a/smnp/token/tools.py +++ b/smnp/token/tools.py @@ -1,6 +1,8 @@ import re + from smnp.token.model import Token + def tokenizeChar(type, char, input, current, line): if input[current] == char: return (1, Token(type, input[current], (line, current))) @@ -15,6 +17,13 @@ def tokenizeRegexPattern(type, pattern, input, current, line): consumedChars += 1 return (consumedChars, Token(type, value, (line, current)) if consumedChars > 0 else None) +def tokenizeKeywords(type, input, current, line, *keywords): + for keyword in keywords: + result = tokenizeKeyword(type, keyword, input, current, line) + if result[0] > 0: + return result + return (0, None) + def tokenizeKeyword(type, keyword, input, current, line): if len(input) >= current+len(keyword) and input[current:current+len(keyword)] == keyword: return (len(keyword), Token(type, keyword, (line, current))) diff --git a/smnp/token/type.py b/smnp/token/type.py index 746906f..cb691f5 100644 --- a/smnp/token/type.py +++ b/smnp/token/type.py @@ -1,23 +1,24 @@ -from enum import Enum +from enum import Enum, auto + class TokenType(Enum): - OPEN_PAREN = 1 - CLOSE_PAREN = 2 - ASTERISK = 3 - STRING = 4 - IDENTIFIER = 5 - COMMA = 6 - INTEGER = 7 - OPEN_BRACKET = 8 - CLOSE_BRACKET = 9 - ASSIGN = 10 - COLON = 11 - NOTE = 12 - COMMENT = 13 - PERCENT = 14 - MINUS = 15 - FUNCTION = 16 - RETURN = 17 - DOT = 18 - OPEN_SQUARE = 19 - CLOSE_SQUARE = 20 + OPEN_PAREN = auto() + CLOSE_PAREN = auto() + ASTERISK = auto() + STRING = auto() + IDENTIFIER = auto() + COMMA = auto() + INTEGER = auto() + OPEN_BRACKET = auto() + CLOSE_BRACKET = auto() + ASSIGN = auto() + NOTE = auto() + COMMENT = auto() + PERCENT = auto() + MINUS = auto() + FUNCTION = auto() + RETURN = auto() + DOT = auto() + OPEN_SQUARE = auto() + CLOSE_SQUARE = auto() + TYPE = auto()