Add important todo to tokenizer

2019-07-06 12:40:21 +02:00
parent 7b4f4fa8fb
commit 675b1774fe
2 changed files with 26 additions and 24 deletions
--- a/smnp/token/tokenizer.py
+++ b/smnp/token/tokenizer.py
@@ -19,6 +19,11 @@ from smnp.token.tokenizers.string import tokenizeString
 from smnp.token.tokenizers.whitespace import tokenizeWhitespaces
 from smnp.token.type import TokenType

+# TODO !!!
+# Enable tokenizer to detect separators of tokens
+# for example, "notes" instead of being tokenized
+# to IDENTIFIER(notes) is tokenized to [TYPE(note), IDENTIFIER(s)]
+
 tokenizers = (
    tokenizeOpenParen, 
    tokenizeCloseParen,