Add important todo to tokenizer

grammar

@@ -5,36 +5,33 @@ CHAR = ... \ '"'
 PITCH = 'c' | 'd' | 'e' | 'f' | 'g' | 'a' | 'h'
 PITCH_MODIFIER = 'b' | '#'
 
-integer := '-' DIGIT+ | DIGIT+
+<integer> := '-' DIGIT+ | DIGIT+
-string := '"' CHAR* '"'
+<string> := '"' CHAR* '"'
-note := '@' PITCH PITCH_MODIFIER? DIGIT? ['.' DIGIT+ 'd'?]?
+<note> := '@' PITCH PITCH_MODIFIER? DIGIT? ['.' DIGIT+ 'd'?]?
-identifier := ID [ID|DIGIT]*
+<identifier> := ID [ID|DIGIT]*
 
 # Parser
-expr := integer accessTail | integer
+<expr> := <integer> <accessTail> | <integer>
-expr := string accessTail | string
+<expr> := <string> <accessTail> | <string>
-expr := note accessTail | note
+<expr> := <note> <accessTail> | <note>
-expr := identifier accessTail | identifier '=' expr | functionCall | identifier
+<expr> := <identifier> <accessTail> | <identifier> '=' <expr> | <identifier> <list> | <identifier>
-expr := list accessTail | list
+<expr> := <list> <accessTail> | <list>
-expr := functionCall accessTail | functionCall
 
-functionCall := identifier list
+<accessTail> := '.' <expr> <accessTail> | e
 
-accessTail := '.' expr accessTail | e
+<list> := '[' ']' | '[' <expr> <listTail>
+<listTail> := <expr> ', ' <listTail> | ']'
 
-list := '[' ']' | '[' expr listTail
+<arglist> := '(' ')' | '(' <expr> <arglistTail>
-listTail := expr ', ' listTail | ']'
+<arglistTail> := <expr> ', ' <arglistTail> | ')'
 
-argList := '(' ')' | '(' expr argListTail
+block := '{' <stmt>* '}'
-argListTail := expr ', ' argListTail | ')'
 
-block := '{' stmt* '}'
+<stmt> := <expr> <asteriskTail> | <expr> #nie wiem czy zamiast '<expr> <asteriskTail>' nie powinno być wprost co może wyprodukować iterator dla asterisk
+<asteriskTail> := '*' <stmt> | e
+<stmt> := <block>
+<stmt> := 'return' <expr>
+<stmt> := 'function' <identifier> <list> block
 
-stmt := expr asteriskTail | expr #nie wiem czy zamiast 'expr asteriskTail' nie powinno być wprost co może wyprodukować iterator dla asterisk
+<program> := <stmt>*
-asteriskTail := '*' stmt | e
-stmt := block
-stmt := 'return' expr
-stmt := 'function' identifier list block
-
-program := stmt*
 

smnp/token/tokenizer.py

@@ -19,6 +19,11 @@ from smnp.token.tokenizers.string import tokenizeString
 from smnp.token.tokenizers.whitespace import tokenizeWhitespaces
 from smnp.token.type import TokenType
 
+# TODO !!!
+# Enable tokenizer to detect separators of tokens
+# for example, "notes" instead of being tokenized
+# to IDENTIFIER(notes) is tokenized to [TYPE(note), IDENTIFIER(s)]
+
 tokenizers = (
     tokenizeOpenParen, 
     tokenizeCloseParen,