Improve errors raising by parser

smnp/ast/node/access.py

@@ -1,6 +1,7 @@
 from smnp.ast.node.expression import ExpressionNode
 from smnp.ast.node.ignore import IgnoredNode
 from smnp.ast.parser import Parser
+from smnp.error.syntax import SyntaxException
 from smnp.token.type import TokenType
 
 
@@ -50,5 +51,6 @@ class AccessNode(ExpressionNode):
 
         return Parser.oneOf(
             IdentifierNode._literalParser(),
-            IdentifierNode._functionCallParser()
+            IdentifierNode._functionCallParser(),
+            exception=lambda input: SyntaxException(f"Expected property name or method call, found '{input.current().rawValue}'", input.currentPos())
         )

smnp/ast/node/block.py

@@ -14,7 +14,7 @@ class BlockNode(StatementNode):
 
         return Parser.loop(
             Parser.terminalParser(TokenType.OPEN_BRACKET),
-            StatementNode.parse,
+            Parser.doAssert(StatementNode.parse, f"statement or '{TokenType.CLOSE_BRACKET.key}'"),
             Parser.terminalParser(TokenType.CLOSE_BRACKET),
-            createNode=createNode
+            createNode=createNode,
         )(input)

smnp/ast/node/expression.py

@@ -44,7 +44,7 @@ class ExpressionNode(Node):
         return Parser.allOf(
             cls._expressionParser(),
             Parser.terminalParser(TokenType.ASTERISK),
-            StatementNode.parse,
+            Parser.doAssert(StatementNode.parse, 'statement'),
             createNode=createNode
         )
 

smnp/ast/node/extend.py

@@ -48,10 +48,10 @@ class ExtendNode(StatementNode):
 
         return Parser.allOf(
             Parser.terminalParser(TokenType.EXTEND),
-            TypeNode.parse,
+            Parser.doAssert(TypeNode.parse, "type being extended"),
-            Parser.terminalParser(TokenType.AS),
+            Parser.terminalParser(TokenType.AS, doAssert=True),
-            IdentifierNode.identifierParser(),
+            Parser.doAssert(IdentifierNode.identifierParser(), "variable name"),
-            cls._methodsDeclarationsParser(),
+            Parser.doAssert(cls._methodsDeclarationsParser(), "methods declarations"),
             createNode=createNode
         )(input)
 
@@ -64,7 +64,7 @@ class ExtendNode(StatementNode):
 
         return Parser.loop(
             Parser.terminalParser(TokenType.OPEN_BRACKET),
-            FunctionDefinitionNode.parse,
+            Parser.doAssert(FunctionDefinitionNode.parse, f"method declaration or '{TokenType.CLOSE_BRACKET.key}'"),
             Parser.terminalParser(TokenType.CLOSE_BRACKET),
             createNode=createNode
         )

smnp/ast/node/function.py

@@ -56,9 +56,9 @@ class FunctionDefinitionNode(StatementNode):
 
         return Parser.allOf(
             Parser.terminalParser(TokenType.FUNCTION),
-            IdentifierNode.identifierParser(),
+            Parser.doAssert(IdentifierNode.identifierParser(), "function name"),
-            cls._argumentsDeclarationParser(),
+            Parser.doAssert(cls._argumentsDeclarationParser(), "arguments list"),
-            BlockNode.parse,
+            Parser.doAssert(BlockNode.parse, "function body"),
             createNode=createNode
         )(input)
 

smnp/ast/node/identifier.py

@@ -30,7 +30,7 @@ class IdentifierNode(AccessNode):
         return Parser.allOf(
             IdentifierNode.identifierParser(),
             Parser.terminalParser(TokenType.ASSIGN),
-            ExpressionNode.parse,
+            Parser.doAssert(ExpressionNode.parse, "expression"),
             createNode=createNode
         )
 

smnp/ast/node/imports.py

@@ -55,10 +55,10 @@ class ImportNode(Node):
         return Parser.allOf(
             Parser.terminalParser(TokenType.IMPORT),
             TypeNode.parse,
-            Parser.terminalParser(TokenType.FROM),
+            Parser.doAssert(Parser.terminalParser(TokenType.FROM), "'from <source> as <variable name>'"),
-            StringLiteralNode._literalParser(),
+            Parser.doAssert(StringLiteralNode._literalParser(), "source as a string"),
-            Parser.terminalParser(TokenType.AS),
+            Parser.doAssert(Parser.terminalParser(TokenType.AS), "'as <variable name>'"),
-            IdentifierNode.identifierParser(),
+            Parser.doAssert(IdentifierNode.identifierParser(), "variable name"),
             createNode=createNode
         )
 
@@ -71,6 +71,6 @@ class ImportNode(Node):
 
         return Parser.allOf(
             Parser.terminalParser(TokenType.IMPORT),
-            StringLiteralNode._literalParser(),
+            Parser.doAssert(StringLiteralNode._literalParser(), "source as a string"),
             createNode=createNode
         )

smnp/ast/node/integer.py

@@ -22,7 +22,7 @@ class IntegerLiteralNode(AccessNode):
 
         return Parser.allOf(
             Parser.terminalParser(TokenType.MINUS),
-            cls._positiveIntegerParser(),
+            Parser.doAssert(cls._positiveIntegerParser(), "integer"),
             createNode=createNode
         )
 

smnp/ast/node/invocation.py

@@ -2,6 +2,7 @@ from smnp.ast.node.access import AccessNode
 from smnp.ast.node.expression import ExpressionNode
 from smnp.ast.node.iterable import abstractIterableParser
 from smnp.ast.node.model import Node
+from smnp.ast.parser import Parser
 from smnp.token.type import TokenType
 
 
@@ -9,7 +10,8 @@ class ArgumentsListNode(Node):
 
     @classmethod
     def _parse(cls, input):
-        return abstractIterableParser(ArgumentsListNode, TokenType.OPEN_PAREN, TokenType.CLOSE_PAREN, ExpressionNode.parse)(input)
+        return abstractIterableParser(ArgumentsListNode, TokenType.OPEN_PAREN, TokenType.CLOSE_PAREN,
+                                      Parser.doAssert(ExpressionNode.parse, "expression"))(input)
 
 
 class FunctionCall(AccessNode):

smnp/ast/node/iterable.py

@@ -41,7 +41,7 @@ def abstractIterableParser(iterableNodeType, openTokenType, closeTokenType, item
                 return node
 
             return Parser.allOf(
-                Parser.terminalParser(TokenType.COMMA),
+                Parser.terminalParser(TokenType.COMMA, doAssert=True),
                 itemParser,
                 AbstractIterableTailNode.parse,
                 createNode=createNode

smnp/ast/node/list.py

@@ -1,6 +1,7 @@
 from smnp.ast.node.access import AccessNode
 from smnp.ast.node.expression import ExpressionNode
 from smnp.ast.node.iterable import abstractIterableParser
+from smnp.ast.parser import Parser
 from smnp.token.type import TokenType
 
 
@@ -8,4 +9,5 @@ class ListNode(AccessNode):
 
     @classmethod
     def _literalParser(cls):
-        return abstractIterableParser(ListNode, TokenType.OPEN_SQUARE, TokenType.CLOSE_SQUARE, ExpressionNode.parse)
+        return abstractIterableParser(ListNode, TokenType.OPEN_SQUARE, TokenType.CLOSE_SQUARE,
+                                      Parser.doAssert(ExpressionNode.parse, "expression"))

smnp/ast/node/program.py

@@ -21,7 +21,7 @@ class Program(Node):
                 ExpressionNode.parse,
                 ImportNode.parse,
                 StatementNode.parse,
-                exception = SyntaxException(f"Unknown statement: {input.current().pos}")
+                exception = SyntaxException(f"Invalid statement: {input.currentToEndOfLine()}", input.current().pos)
             )(input)
 
         root = Program()

smnp/ast/node/ret.py

@@ -27,6 +27,6 @@ class ReturnNode(StatementNode):
 
         return Parser.allOf(
             Parser.terminalParser(TokenType.RETURN),
-            ExpressionNode.parse,
+            Parser.doAssert(ExpressionNode.parse, "expression"),
             createNode=createNode
         )(input)

smnp/ast/node/variable.py

@@ -36,7 +36,7 @@ class TypedVariableNode(ExpressionNode):
 
         return Parser.allOf(
             Parser.terminalParser(TokenType.TYPE, lambda val, pos: TypeNode.withValue(val, pos)),
-            IdentifierNode.identifierParser(),
+            Parser.doAssert(IdentifierNode.identifierParser(), "variable name"),
             createNode=createNode
         )
 

smnp/ast/parser.py

@@ -1,5 +1,6 @@
 from smnp.ast.node.ignore import IgnoredNode
 from smnp.ast.node.model import ParseResult, Node
+from smnp.error.syntax import SyntaxException
 
 
 def parse(input):
@@ -11,7 +12,7 @@ class Parser:
 
     # a -> A
     @staticmethod
-    def terminalParser(expectedType, createNode=None):
+    def terminalParser(expectedType, createNode=None, doAssert=False):
         def provideNode(value, pos):
             if createNode is None:
                 return IgnoredNode(pos)
@@ -22,6 +23,10 @@ class Parser:
                 token = input.current()
                 input.ahead()
                 return ParseResult.OK(provideNode(token.value, token.pos))
+            elif doAssert:
+                found = f", found '{input.current().rawValue}'" if input.hasCurrent() else ""
+                raise SyntaxException(f"Expected '{expectedType.key}'{found}", input.currentPos())
+
             return ParseResult.FAIL()
 
         return parse
@@ -37,7 +42,11 @@ class Parser:
                     return value
 
             if exception is not None:
-                raise exception
+                if callable(exception):
+                    raise exception(input)
+                else:
+                    raise exception
+
 
             input.reset(snap)
             return ParseResult.FAIL()
@@ -60,7 +69,10 @@ class Parser:
 
                 if not result.result:
                     if exception is not None:
-                        raise exception
+                        if callable(exception):
+                            raise exception(input)
+                        else:
+                            raise exception
 
                     input.reset(snap)
                     return ParseResult.FAIL()
@@ -114,3 +126,16 @@ class Parser:
 
         return parse
 
+    @staticmethod
+    def doAssert(parser, expected):
+        def parse(input):
+            result = parser(input)
+
+            if not result.result:
+                found = f", found '{input.current().rawValue}'" if input.hasCurrent() else ''
+
+                raise SyntaxException(f"Expected {expected}{found}", input.currentPos())
+
+            return result
+
+        return parse

smnp/ast/tools.py

@@ -5,4 +5,4 @@ def assertToken(expected, input):
     if not input.hasCurrent():
         raise SyntaxException(f"Expected '{expected}'")
     if expected != input.current().type:
-        raise SyntaxException(f"Expected '{expected}', found '{input.current().value}'", input.current().pos)
+        raise SyntaxException(f"Expected '{expected}', found '{input.current().rawValue}'", input.current().pos)

smnp/main.py

@@ -15,9 +15,11 @@ def main():
         tokens = tokenize(lines)
 
         ast = parse(tokens)
+
+
         ast.print()
 
-
+        sys.exit(0)
         env = createEnvironment()
 
         evaluate(ast, env)

smnp/runtime/evaluator.py

@@ -1,57 +1,2 @@
-from smnp.ast.node.access import AccessNode
-from smnp.ast.node.assignment import AssignmentNode
-from smnp.ast.node.asterisk import AsteriskNode
-from smnp.ast.node.block import BlockNode
-from smnp.ast.node.colon import ColonNode
-from smnp.ast.node.function import FunctionDefinitionNode, FunctionCallNode
-from smnp.ast.node.identifier import IdentifierNode
-from smnp.ast.node.integer import IntegerLiteralNode
-from smnp.ast.node.list import ListNode
-from smnp.ast.node.note import NoteLiteralNode
-from smnp.ast.node.percent import PercentNode
-from smnp.ast.node.program import Program
-from smnp.ast.node.string import StringLiteralNode
-
 def evaluate(input, environment):
-    from smnp.runtime.evaluators.access import evaluateAccess
+    pass
-    from smnp.runtime.evaluators.assignment import evaluateAssignment
-    from smnp.runtime.evaluators.asterisk import evaluateAsterisk
-    from smnp.runtime.evaluators.block import evaluateBlock
-    from smnp.runtime.evaluators.colon import evaluateColon
-    from smnp.runtime.evaluators.function import evaluateFunctionDefinition, evaluateFunctionCall
-    from smnp.runtime.evaluators.identifier import evaluateIdentifier
-    from smnp.runtime.evaluators.integer import evaluateInteger
-    from smnp.runtime.evaluators.list import evaluateList
-    from smnp.runtime.evaluators.note import evaluateNote
-    from smnp.runtime.evaluators.percent import evaluatePercent
-    from smnp.runtime.evaluators.program import evaluateProgram
-    from smnp.runtime.evaluators.string import evaluateString
-
-    if isinstance(input, Program):
-        return evaluateProgram(input, environment)
-    if isinstance(input, IntegerLiteralNode):
-        return evaluateInteger(input, environment)
-    if isinstance(input, PercentNode):
-        return evaluatePercent(input, environment)
-    if isinstance(input, StringLiteralNode):
-        return evaluateString(input, environment)
-    if isinstance(input, NoteLiteralNode):
-        return evaluateNote(input, environment)
-    if isinstance(input, FunctionDefinitionNode):
-        return evaluateFunctionDefinition(input, environment)
-    if isinstance(input, FunctionCallNode):
-        return evaluateFunctionCall(input, environment)
-    if isinstance(input, AccessNode):
-        return evaluateAccess(input, environment)
-    if isinstance(input, BlockNode):
-        return evaluateBlock(input, environment)
-    if isinstance(input, ListNode):
-        return evaluateList(input, environment)
-    if isinstance(input, AssignmentNode):
-        return evaluateAssignment(input, environment)
-    if isinstance(input, AsteriskNode):
-        return evaluateAsterisk(input, environment)
-    if isinstance(input, ColonNode):
-        return evaluateColon(input, environment)
-    if isinstance(input, IdentifierNode):
-        return evaluateIdentifier(input, environment)

smnp/token/model.py

@@ -1,19 +1,25 @@
 class Token:
-    def __init__(self, type, value, pos):
+    def __init__(self, type, value, pos, rawValue=None):
         self.type = type
         self.value = value    
         self.pos = pos
+        if rawValue is None:
+            rawValue = value
+        self.rawValue = rawValue
+
     def __str__(self):
         return "Token(" + str(self.type) + ", '" + str(self.value) + "', " + str(self.pos) + ")"
+
     def __repr__(self):
         return self.__str__()
 
 
 class TokenList:
-    def __init__(self, tokens = []):
+    def __init__(self, tokens, lines):
         self.tokens = tokens
         self.cursor = 0
         self.snap = 0
+        self.lines = lines
         
     def append(self, token):
         self.tokens.append(token)
@@ -26,6 +32,10 @@ class TokenList:
             raise RuntimeError(f"Cursor points to not existing token! Cursor = {self.cursor}, len = {len(self.tokens)}")
         return self.tokens[self.cursor]
 
+    def currentPos(self):
+        #TODO maybe change raw pos (position) tuple to some class with method "nextCol()", "nextRow()" etc.
+        return self.current().pos if self.hasCurrent() else (self.tokens[-1].pos[0], self.tokens[-1].pos[1]+1) if len(self.tokens) > 0 else None
+
     def isCurrent(self, type):
         return self.hasCurrent() and self.current().type == type
 
@@ -49,7 +59,10 @@ class TokenList:
         
     def reset(self, snap):
         self.cursor = snap
-    
+
+    def currentToEndOfLine(self):
+        return self.lines[self.current().pos[0]][self.current().pos[1]:]
+
     def __str__(self):
         return f"[Current({self.cursor}): {self.current() if self.hasCurrent() else 'out of tokens'}\n{', '.join([str(token) for token in self.tokens])}]"
     

smnp/token/tokenizer.py

@@ -70,7 +70,7 @@ def tokenize(lines):
             current += consumedChars
             tokens.append(token)
             
-    return TokenList(filterTokens(filters, tokens))
+    return TokenList(filterTokens(filters, tokens), lines)
 
 
 def combinedTokenizer(line, current, lineNumber):

smnp/token/tokenizers/note.py

@@ -11,35 +11,43 @@ def tokenizeNote(input, current, line):
     octave = None
     duration = None
     dot = False
+    rawValue = ''
     if input[current] == '@':
+        rawValue += input[current+consumedChars]
         consumedChars += 1
         if input[current+consumedChars] in ('C', 'c', 'D', 'd', 'E', 'e', 'F', 'f', 'G', 'g', 'A', 'a', 'H', 'h', 'B', 'b'):
+            rawValue += input[current + consumedChars]
             notePitch = input[current+consumedChars]
             consumedChars += 1
             
-            if current+consumedChars < len(input) and input[current+consumedChars] in ('b', '#'):                        
+            if current+consumedChars < len(input) and input[current+consumedChars] in ('b', '#'):
+                rawValue += input[current + consumedChars]
                 notePitch += input[current+consumedChars]
                 consumedChars += 1
                 
-            if current+consumedChars < len(input) and re.match(r'\d', input[current+consumedChars]):            
+            if current+consumedChars < len(input) and re.match(r'\d', input[current+consumedChars]):
+                rawValue += input[current + consumedChars]
                 octave = input[current+consumedChars]
                 consumedChars += 1
                 
             if current+consumedChars < len(input) and input[current+consumedChars] == ':':
+                rawValue += input[current + consumedChars]
                 duration = ''
                 consumedChars += 1
                 while current+consumedChars < len(input) and re.match(r'\d', input[current+consumedChars]):
+                    rawValue += input[current + consumedChars]
                     duration += input[current+consumedChars]        
                     consumedChars += 1
                 if len(duration) == 0:
                     return (0, None)
                 dot = (current+consumedChars) < len(input) and input[current+consumedChars] == 'd'
                 if dot:
+                    rawValue += input[current + consumedChars]
                     consumedChars += 1
 
             octave = int(octave) if octave is not None else None
             duration = int(duration) if duration is not None else None
             value = Note(notePitch, octave, duration, dot)
 
-            return (consumedChars, Token(TokenType.NOTE, value, (line, current)))
+            return (consumedChars, Token(TokenType.NOTE, value, (line, current), rawValue))
     return (0, None)

smnp/token/type.py

@@ -1,28 +1,36 @@
-from enum import Enum, auto
+from enum import Enum
 
 
 class TokenType(Enum):
-    OPEN_PAREN = auto()
+    OPEN_PAREN = '('
-    CLOSE_PAREN = auto()
+    CLOSE_PAREN = ')'
-    ASTERISK = auto()
+    ASTERISK = '*'
-    STRING = auto()
+    STRING = 'string'
-    IDENTIFIER = auto()
+    IDENTIFIER = 'identifier'
-    COMMA = auto()
+    COMMA = ','
-    INTEGER = auto()
+    INTEGER = 'integer'
-    OPEN_BRACKET = auto()
+    OPEN_BRACKET = '{'
-    CLOSE_BRACKET = auto()
+    CLOSE_BRACKET = '}'
-    ASSIGN = auto()
+    ASSIGN = '='
-    NOTE = auto()
+    NOTE = 'note'
-    COMMENT = auto()
+    COMMENT = 'comment'
-    PERCENT = auto()
+    PERCENT = 'percent'
-    MINUS = auto()
+    MINUS = '-'
-    FUNCTION = auto()
+    FUNCTION = 'function'
-    RETURN = auto()
+    RETURN = 'return'
-    DOT = auto()
+    DOT = '.'
-    OPEN_SQUARE = auto()
+    OPEN_SQUARE = '['
-    CLOSE_SQUARE = auto()
+    CLOSE_SQUARE = ']'
-    TYPE = auto()
+    TYPE = 'type'
-    EXTEND = auto()
+    EXTEND = 'extend'
-    IMPORT = auto()
+    IMPORT = 'import'
-    FROM = auto()
+    FROM = 'from'
-    AS = auto()
+    AS = 'as'
+
+    @property
+    def key(self):
+        return self.value
+
+    @key.setter
+    def key(self, value):
+        raise RuntimeError("Cannot change key of token type")