Improve float type tokenizer

smnp/token/tokenizers/float.py

@@ -1,25 +1,17 @@
 from smnp.token.model import Token
-from smnp.token.tools import regexPatternTokenizer, keywordTokenizer
+from smnp.token.tools import regexPatternTokenizer, keywordTokenizer, allOf
 from smnp.token.type import TokenType
 
 
-def floatTokenizer(input, current, line):
+def createToken(pos, beforeDot, dot, afterDot):
-    consumedChars = 0
+    rawValue = f"{beforeDot.value}.{afterDot.value}"
-    value = ""
+    value = float(rawValue)
-    consumed, token = regexPatternTokenizer(TokenType.INTEGER, r'\d')(input, current, line)
+    return Token(TokenType.FLOAT, value, pos, rawValue)
-    if consumed > 0:
-        consumedChars += consumed
-        value += token.value
-        consumed, token = keywordTokenizer(TokenType.DOT, ".")(input, current+consumedChars, line)
-        if consumed > 0:
-            consumedChars += consumed
-            value += token.value
-            consumed, token = regexPatternTokenizer(TokenType.INTEGER, r'\d')(input, current+consumedChars, line)
-            if consumed > 0:
-                consumedChars += consumed
-                value += token.value
-                print(value)
-                return (consumedChars, Token(TokenType.FLOAT, float(value), (current, line), value))
 
 
-    return (0, None)
+floatTokenizer = allOf(
+    regexPatternTokenizer(TokenType.INTEGER, r'\d'),
+    keywordTokenizer(None, "."),
+    regexPatternTokenizer(TokenType.INTEGER, r'\d'),
+    createToken=createToken
+)

smnp/token/tools.py

@@ -61,3 +61,19 @@ def mapValue(tokenizer, mapper):
         return (0, None)
 
     return tokenize
+
+def allOf(*tokenizers, createToken):
+    def combinedTokenizer(input, current, line):
+        consumedChars = 0
+        tokens = []
+        for tokenizer in tokenizers:
+            consumed, token = tokenizer(input, current+consumedChars, line)
+            if consumed > 0:
+                consumedChars += consumed
+                tokens.append(token)
+            else:
+                return (0, None)
+
+        return (consumedChars, createToken((current, line), *tokens))
+
+    return combinedTokenizer