Add new token: TokenType.BOOL

smnp/token/model.py

@@ -8,7 +8,7 @@ class Token:
         self.rawValue = rawValue
 
     def __str__(self):
-        return "Token(" + str(self.type) + ", '" + str(self.value) + "', " + str(self.pos) + ")"
+        return "{" + str(self.type.name) + ", '" + str(self.value) + "', " + str(self.pos) + "}"
 
     def __repr__(self):
         return self.__str__()

smnp/token/tokenizer.py

@@ -1,5 +1,6 @@
 from smnp.error.syntax import SyntaxException
 from smnp.token.model import TokenList
+from smnp.token.tokenizers.bool import boolTokenizer
 from smnp.token.tokenizers.comment import commentTokenizer
 from smnp.token.tokenizers.identifier import identifierTokenizer
 from smnp.token.tokenizers.keyword import typeTokenizer
@@ -31,8 +32,9 @@ tokenizers = (
     # Types
     separated(regexPatternTokenizer(TokenType.INTEGER, r'\d')),
     stringTokenizer,
-    typeTokenizer,
     noteTokenizer,
+    boolTokenizer,
+    typeTokenizer,
 
     # Keywords
     separated(defaultTokenizer(TokenType.FUNCTION)),
@@ -50,7 +52,6 @@ tokenizers = (
     commentTokenizer,
 )
 
-
 filters = [
     lambda token: token.type is not None,
     lambda token: token.type != TokenType.COMMENT
@@ -58,33 +59,33 @@ filters = [
 
 
 def tokenize(lines):
-    tokens = []     
-    for lineNumber, line in enumerate(lines):    
+    tokens = []
+    for lineNumber, line in enumerate(lines):
         current = 0
         while current < len(line):
-            consumedChars, token = combinedTokenizer(line, current, lineNumber)            
-            
+            consumedChars, token = combinedTokenizer(line, current, lineNumber)
+
             if consumedChars == 0:
                 raise SyntaxException(f"Unknown symbol '{line[current]}'", (lineNumber, current))
-            
+
             current += consumedChars
             tokens.append(token)
-            
+
     return TokenList(filterTokens(filters, tokens), lines)
 
 
 def combinedTokenizer(line, current, lineNumber):
     for tokenizer in tokenizers:
         consumedChars, token = tokenizer(line, current, lineNumber)
-        if consumedChars > 0:                      
+        if consumedChars > 0:
             return (consumedChars, token)
     return (0, None)
 
 
-def filterTokens(filters, tokens):    
+def filterTokens(filters, tokens):
     if not filters:
         return tokens
-    
+
     return list(filterTokens(filters[1:], (token for token in tokens if filters[0](token))))
 
 

smnp/token/tokenizers/bool.py

@@ -0,0 +1,11 @@
+from smnp.token.tools import keywordsTokenizer, separated
+from smnp.token.type import TokenType
+
+
+def boolTokenizer(input, current, line):
+    consumedChars, token = separated(keywordsTokenizer(TokenType.BOOL, "true", "false"))(input, current, line)
+    if consumedChars > 0:
+        token.value = token.value == "true"
+        return (consumedChars, token)
+
+    return (0, None)

smnp/token/type.py

@@ -21,6 +21,7 @@ class TokenType(Enum):
     INTEGER = 'integer'
     STRING = 'string'
     NOTE = 'note'
+    BOOL = 'bool'
     TYPE = 'type'
     FUNCTION = 'function'
     RETURN = 'return'