From 675b1774feb998203d0f892f003942334cf95d10 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bart=C5=82omiej=20Pluta?= <the.orleander@gmail.com>
Date: Sat, 6 Jul 2019 12:40:21 +0200
Subject: [PATCH] Add important todo to tokenizer

---
 grammar                 | 45 +++++++++++++++++++----------------------
 smnp/token/tokenizer.py |  5 +++++
 2 files changed, 26 insertions(+), 24 deletions(-)

diff --git a/grammar b/grammar
index a50250a..8350625 100644
--- a/grammar
+++ b/grammar
@@ -5,36 +5,33 @@ CHAR = ... \ '"'
 PITCH = 'c' | 'd' | 'e' | 'f' | 'g' | 'a' | 'h'
 PITCH_MODIFIER = 'b' | '#'
 
-integer := '-' DIGIT+ | DIGIT+
-string := '"' CHAR* '"'
-note := '@' PITCH PITCH_MODIFIER? DIGIT? ['.' DIGIT+ 'd'?]?
-identifier := ID [ID|DIGIT]*
+<integer> := '-' DIGIT+ | DIGIT+
+<string> := '"' CHAR* '"'
+<note> := '@' PITCH PITCH_MODIFIER? DIGIT? ['.' DIGIT+ 'd'?]?
+<identifier> := ID [ID|DIGIT]*
 
 # Parser
-expr := integer accessTail | integer
-expr := string accessTail | string
-expr := note accessTail | note
-expr := identifier accessTail | identifier '=' expr | functionCall | identifier
-expr := list accessTail | list
-expr := functionCall accessTail | functionCall
+<expr> := <integer> <accessTail> | <integer>
+<expr> := <string> <accessTail> | <string>
+<expr> := <note> <accessTail> | <note>
+<expr> := <identifier> <accessTail> | <identifier> '=' <expr> | <identifier> <list> | <identifier>
+<expr> := <list> <accessTail> | <list>
 
-functionCall := identifier list
+<accessTail> := '.' <expr> <accessTail> | e
 
-accessTail := '.' expr accessTail | e
+<list> := '[' ']' | '[' <expr> <listTail>
+<listTail> := <expr> ', ' <listTail> | ']'
 
-list := '[' ']' | '[' expr listTail
-listTail := expr ', ' listTail | ']'
+<arglist> := '(' ')' | '(' <expr> <arglistTail>
+<arglistTail> := <expr> ', ' <arglistTail> | ')'
 
-argList := '(' ')' | '(' expr argListTail
-argListTail := expr ', ' argListTail | ')'
+block := '{' <stmt>* '}'
 
-block := '{' stmt* '}'
+<stmt> := <expr> <asteriskTail> | <expr> #nie wiem czy zamiast '<expr> <asteriskTail>' nie powinno być wprost co może wyprodukować iterator dla asterisk
+<asteriskTail> := '*' <stmt> | e
+<stmt> := <block>
+<stmt> := 'return' <expr>
+<stmt> := 'function' <identifier> <list> block
 
-stmt := expr asteriskTail | expr #nie wiem czy zamiast 'expr asteriskTail' nie powinno być wprost co może wyprodukować iterator dla asterisk
-asteriskTail := '*' stmt | e
-stmt := block
-stmt := 'return' expr
-stmt := 'function' identifier list block
-
-program := stmt*
+<program> := <stmt>*
 
diff --git a/smnp/token/tokenizer.py b/smnp/token/tokenizer.py
index d884784..622f28e 100644
--- a/smnp/token/tokenizer.py
+++ b/smnp/token/tokenizer.py
@@ -19,6 +19,11 @@ from smnp.token.tokenizers.string import tokenizeString
 from smnp.token.tokenizers.whitespace import tokenizeWhitespaces
 from smnp.token.type import TokenType
 
+# TODO !!!
+# Enable tokenizer to detect separators of tokens
+# for example, "notes" instead of being tokenized
+# to IDENTIFIER(notes) is tokenized to [TYPE(note), IDENTIFIER(s)]
+
 tokenizers = (
     tokenizeOpenParen, 
     tokenizeCloseParen,