Improve errors raising by parser

smnp/token/model.py

@@ -1,19 +1,25 @@
 class Token:
-    def __init__(self, type, value, pos):
+    def __init__(self, type, value, pos, rawValue=None):
         self.type = type
         self.value = value    
         self.pos = pos
+        if rawValue is None:
+            rawValue = value
+        self.rawValue = rawValue
+
     def __str__(self):
         return "Token(" + str(self.type) + ", '" + str(self.value) + "', " + str(self.pos) + ")"
+
     def __repr__(self):
         return self.__str__()
 
 
 class TokenList:
-    def __init__(self, tokens = []):
+    def __init__(self, tokens, lines):
         self.tokens = tokens
         self.cursor = 0
         self.snap = 0
+        self.lines = lines
         
     def append(self, token):
         self.tokens.append(token)
@@ -26,6 +32,10 @@ class TokenList:
             raise RuntimeError(f"Cursor points to not existing token! Cursor = {self.cursor}, len = {len(self.tokens)}")
         return self.tokens[self.cursor]
 
+    def currentPos(self):
+        #TODO maybe change raw pos (position) tuple to some class with method "nextCol()", "nextRow()" etc.
+        return self.current().pos if self.hasCurrent() else (self.tokens[-1].pos[0], self.tokens[-1].pos[1]+1) if len(self.tokens) > 0 else None
+
     def isCurrent(self, type):
         return self.hasCurrent() and self.current().type == type
 
@@ -49,7 +59,10 @@ class TokenList:
         
     def reset(self, snap):
         self.cursor = snap
-    
+
+    def currentToEndOfLine(self):
+        return self.lines[self.current().pos[0]][self.current().pos[1]:]
+
     def __str__(self):
         return f"[Current({self.cursor}): {self.current() if self.hasCurrent() else 'out of tokens'}\n{', '.join([str(token) for token in self.tokens])}]"
     

smnp/token/tokenizer.py

@@ -70,7 +70,7 @@ def tokenize(lines):
             current += consumedChars
             tokens.append(token)
             
-    return TokenList(filterTokens(filters, tokens))
+    return TokenList(filterTokens(filters, tokens), lines)
 
 
 def combinedTokenizer(line, current, lineNumber):

smnp/token/tokenizers/note.py

@@ -11,35 +11,43 @@ def tokenizeNote(input, current, line):
     octave = None
     duration = None
     dot = False
+    rawValue = ''
     if input[current] == '@':
+        rawValue += input[current+consumedChars]
         consumedChars += 1
         if input[current+consumedChars] in ('C', 'c', 'D', 'd', 'E', 'e', 'F', 'f', 'G', 'g', 'A', 'a', 'H', 'h', 'B', 'b'):
+            rawValue += input[current + consumedChars]
             notePitch = input[current+consumedChars]
             consumedChars += 1
             
-            if current+consumedChars < len(input) and input[current+consumedChars] in ('b', '#'):                        
+            if current+consumedChars < len(input) and input[current+consumedChars] in ('b', '#'):
+                rawValue += input[current + consumedChars]
                 notePitch += input[current+consumedChars]
                 consumedChars += 1
                 
-            if current+consumedChars < len(input) and re.match(r'\d', input[current+consumedChars]):            
+            if current+consumedChars < len(input) and re.match(r'\d', input[current+consumedChars]):
+                rawValue += input[current + consumedChars]
                 octave = input[current+consumedChars]
                 consumedChars += 1
                 
             if current+consumedChars < len(input) and input[current+consumedChars] == ':':
+                rawValue += input[current + consumedChars]
                 duration = ''
                 consumedChars += 1
                 while current+consumedChars < len(input) and re.match(r'\d', input[current+consumedChars]):
+                    rawValue += input[current + consumedChars]
                     duration += input[current+consumedChars]        
                     consumedChars += 1
                 if len(duration) == 0:
                     return (0, None)
                 dot = (current+consumedChars) < len(input) and input[current+consumedChars] == 'd'
                 if dot:
+                    rawValue += input[current + consumedChars]
                     consumedChars += 1
 
             octave = int(octave) if octave is not None else None
             duration = int(duration) if duration is not None else None
             value = Note(notePitch, octave, duration, dot)
 
-            return (consumedChars, Token(TokenType.NOTE, value, (line, current)))
+            return (consumedChars, Token(TokenType.NOTE, value, (line, current), rawValue))
     return (0, None)

smnp/token/type.py

@@ -1,28 +1,36 @@
-from enum import Enum, auto
+from enum import Enum
 
 
 class TokenType(Enum):
-    OPEN_PAREN = auto()
-    CLOSE_PAREN = auto()
-    ASTERISK = auto()
-    STRING = auto()
-    IDENTIFIER = auto()
-    COMMA = auto()
-    INTEGER = auto()
-    OPEN_BRACKET = auto()
-    CLOSE_BRACKET = auto()
-    ASSIGN = auto()
-    NOTE = auto()
-    COMMENT = auto()
-    PERCENT = auto()
-    MINUS = auto()
-    FUNCTION = auto()
-    RETURN = auto()
-    DOT = auto()
-    OPEN_SQUARE = auto()
-    CLOSE_SQUARE = auto()
-    TYPE = auto()
-    EXTEND = auto()
-    IMPORT = auto()
-    FROM = auto()
-    AS = auto()
+    OPEN_PAREN = '('
+    CLOSE_PAREN = ')'
+    ASTERISK = '*'
+    STRING = 'string'
+    IDENTIFIER = 'identifier'
+    COMMA = ','
+    INTEGER = 'integer'
+    OPEN_BRACKET = '{'
+    CLOSE_BRACKET = '}'
+    ASSIGN = '='
+    NOTE = 'note'
+    COMMENT = 'comment'
+    PERCENT = 'percent'
+    MINUS = '-'
+    FUNCTION = 'function'
+    RETURN = 'return'
+    DOT = '.'
+    OPEN_SQUARE = '['
+    CLOSE_SQUARE = ']'
+    TYPE = 'type'
+    EXTEND = 'extend'
+    IMPORT = 'import'
+    FROM = 'from'
+    AS = 'as'
+
+    @property
+    def key(self):
+        return self.value
+
+    @key.setter
+    def key(self, value):
+        raise RuntimeError("Cannot change key of token type")