Create tokenizer for float type

smnp/token/tokenizer.py

@@ -2,6 +2,7 @@ from smnp.error.syntax import SyntaxException
 from smnp.token.model import TokenList
 from smnp.token.tokenizers.bool import boolTokenizer
 from smnp.token.tokenizers.comment import commentTokenizer
+from smnp.token.tokenizers.float import floatTokenizer
 from smnp.token.tokenizers.identifier import identifierTokenizer
 from smnp.token.tokenizers.keyword import typeTokenizer
 from smnp.token.tokenizers.note import noteTokenizer
@@ -41,6 +42,7 @@ tokenizers = (
     defaultTokenizer(TokenType.DOT),
 
     # Types
+    separated(floatTokenizer),
     mapValue(separated(regexPatternTokenizer(TokenType.INTEGER, r'\d')), int),
     stringTokenizer,
     noteTokenizer,

smnp/token/tokenizers/float.py

@@ -0,0 +1,25 @@
+from smnp.token.model import Token
+from smnp.token.tools import regexPatternTokenizer, keywordTokenizer
+from smnp.token.type import TokenType
+
+
+def floatTokenizer(input, current, line):
+    consumedChars = 0
+    value = ""
+    consumed, token = regexPatternTokenizer(TokenType.INTEGER, r'\d')(input, current, line)
+    if consumed > 0:
+        consumedChars += consumed
+        value += token.value
+        consumed, token = keywordTokenizer(TokenType.DOT, ".")(input, current+consumedChars, line)
+        if consumed > 0:
+            consumedChars += consumed
+            value += token.value
+            consumed, token = regexPatternTokenizer(TokenType.INTEGER, r'\d')(input, current+consumedChars, line)
+            if consumed > 0:
+                consumedChars += consumed
+                value += token.value
+                print(value)
+                return (consumedChars, Token(TokenType.FLOAT, float(value), (current, line), value))
+
+
+    return (0, None)

smnp/token/type.py

@@ -30,6 +30,7 @@ class TokenType(Enum):
     NOT = 'not'
     INTEGER = 'integer'
     STRING = 'string'
+    FLOAT = 'float'
     NOTE = 'note'
     BOOL = 'bool'
     TYPE = 'type'