diff --git a/smnp/token/tokenizer.py b/smnp/token/tokenizer.py index 277badb..e00b772 100644 --- a/smnp/token/tokenizer.py +++ b/smnp/token/tokenizer.py @@ -2,6 +2,7 @@ from smnp.error.syntax import SyntaxException from smnp.token.model import TokenList from smnp.token.tokenizers.bool import boolTokenizer from smnp.token.tokenizers.comment import commentTokenizer +from smnp.token.tokenizers.float import floatTokenizer from smnp.token.tokenizers.identifier import identifierTokenizer from smnp.token.tokenizers.keyword import typeTokenizer from smnp.token.tokenizers.note import noteTokenizer @@ -41,6 +42,7 @@ tokenizers = ( defaultTokenizer(TokenType.DOT), # Types + separated(floatTokenizer), mapValue(separated(regexPatternTokenizer(TokenType.INTEGER, r'\d')), int), stringTokenizer, noteTokenizer, diff --git a/smnp/token/tokenizers/float.py b/smnp/token/tokenizers/float.py new file mode 100644 index 0000000..6f05557 --- /dev/null +++ b/smnp/token/tokenizers/float.py @@ -0,0 +1,25 @@ +from smnp.token.model import Token +from smnp.token.tools import regexPatternTokenizer, keywordTokenizer +from smnp.token.type import TokenType + + +def floatTokenizer(input, current, line): + consumedChars = 0 + value = "" + consumed, token = regexPatternTokenizer(TokenType.INTEGER, r'\d')(input, current, line) + if consumed > 0: + consumedChars += consumed + value += token.value + consumed, token = keywordTokenizer(TokenType.DOT, ".")(input, current+consumedChars, line) + if consumed > 0: + consumedChars += consumed + value += token.value + consumed, token = regexPatternTokenizer(TokenType.INTEGER, r'\d')(input, current+consumedChars, line) + if consumed > 0: + consumedChars += consumed + value += token.value + print(value) + return (consumedChars, Token(TokenType.FLOAT, float(value), (current, line), value)) + + + return (0, None) \ No newline at end of file diff --git a/smnp/token/type.py b/smnp/token/type.py index c14d1c5..1812993 100644 --- a/smnp/token/type.py +++ b/smnp/token/type.py @@ -30,6 +30,7 @@ class TokenType(Enum): NOT = 'not' INTEGER = 'integer' STRING = 'string' + FLOAT = 'float' NOTE = 'note' BOOL = 'bool' TYPE = 'type'