diff --git a/smnp/ast/node/access.py b/smnp/ast/node/access.py index 8f7d5a8..8ef0f51 100644 --- a/smnp/ast/node/access.py +++ b/smnp/ast/node/access.py @@ -1,6 +1,7 @@ from smnp.ast.node.expression import ExpressionNode from smnp.ast.node.ignore import IgnoredNode from smnp.ast.parser import Parser +from smnp.error.syntax import SyntaxException from smnp.token.type import TokenType @@ -50,5 +51,6 @@ class AccessNode(ExpressionNode): return Parser.oneOf( IdentifierNode._literalParser(), - IdentifierNode._functionCallParser() + IdentifierNode._functionCallParser(), + exception=lambda input: SyntaxException(f"Expected property name or method call, found '{input.current().rawValue}'", input.currentPos()) ) diff --git a/smnp/ast/node/block.py b/smnp/ast/node/block.py index b9ba7bc..15f255b 100644 --- a/smnp/ast/node/block.py +++ b/smnp/ast/node/block.py @@ -14,7 +14,7 @@ class BlockNode(StatementNode): return Parser.loop( Parser.terminalParser(TokenType.OPEN_BRACKET), - StatementNode.parse, + Parser.doAssert(StatementNode.parse, f"statement or '{TokenType.CLOSE_BRACKET.key}'"), Parser.terminalParser(TokenType.CLOSE_BRACKET), - createNode=createNode + createNode=createNode, )(input) \ No newline at end of file diff --git a/smnp/ast/node/expression.py b/smnp/ast/node/expression.py index 1dec6e8..9a476a0 100644 --- a/smnp/ast/node/expression.py +++ b/smnp/ast/node/expression.py @@ -44,7 +44,7 @@ class ExpressionNode(Node): return Parser.allOf( cls._expressionParser(), Parser.terminalParser(TokenType.ASTERISK), - StatementNode.parse, + Parser.doAssert(StatementNode.parse, 'statement'), createNode=createNode ) diff --git a/smnp/ast/node/extend.py b/smnp/ast/node/extend.py index 0611bb7..5c58714 100644 --- a/smnp/ast/node/extend.py +++ b/smnp/ast/node/extend.py @@ -48,10 +48,10 @@ class ExtendNode(StatementNode): return Parser.allOf( Parser.terminalParser(TokenType.EXTEND), - TypeNode.parse, - Parser.terminalParser(TokenType.AS), - IdentifierNode.identifierParser(), - cls._methodsDeclarationsParser(), + Parser.doAssert(TypeNode.parse, "type being extended"), + Parser.terminalParser(TokenType.AS, doAssert=True), + Parser.doAssert(IdentifierNode.identifierParser(), "variable name"), + Parser.doAssert(cls._methodsDeclarationsParser(), "methods declarations"), createNode=createNode )(input) @@ -64,7 +64,7 @@ class ExtendNode(StatementNode): return Parser.loop( Parser.terminalParser(TokenType.OPEN_BRACKET), - FunctionDefinitionNode.parse, + Parser.doAssert(FunctionDefinitionNode.parse, f"method declaration or '{TokenType.CLOSE_BRACKET.key}'"), Parser.terminalParser(TokenType.CLOSE_BRACKET), createNode=createNode ) \ No newline at end of file diff --git a/smnp/ast/node/function.py b/smnp/ast/node/function.py index 0fe2baf..8d1fdbe 100644 --- a/smnp/ast/node/function.py +++ b/smnp/ast/node/function.py @@ -56,9 +56,9 @@ class FunctionDefinitionNode(StatementNode): return Parser.allOf( Parser.terminalParser(TokenType.FUNCTION), - IdentifierNode.identifierParser(), - cls._argumentsDeclarationParser(), - BlockNode.parse, + Parser.doAssert(IdentifierNode.identifierParser(), "function name"), + Parser.doAssert(cls._argumentsDeclarationParser(), "arguments list"), + Parser.doAssert(BlockNode.parse, "function body"), createNode=createNode )(input) diff --git a/smnp/ast/node/identifier.py b/smnp/ast/node/identifier.py index 7c4bebe..f85ec7c 100644 --- a/smnp/ast/node/identifier.py +++ b/smnp/ast/node/identifier.py @@ -30,7 +30,7 @@ class IdentifierNode(AccessNode): return Parser.allOf( IdentifierNode.identifierParser(), Parser.terminalParser(TokenType.ASSIGN), - ExpressionNode.parse, + Parser.doAssert(ExpressionNode.parse, "expression"), createNode=createNode ) diff --git a/smnp/ast/node/imports.py b/smnp/ast/node/imports.py index ded151b..777ffaf 100644 --- a/smnp/ast/node/imports.py +++ b/smnp/ast/node/imports.py @@ -55,10 +55,10 @@ class ImportNode(Node): return Parser.allOf( Parser.terminalParser(TokenType.IMPORT), TypeNode.parse, - Parser.terminalParser(TokenType.FROM), - StringLiteralNode._literalParser(), - Parser.terminalParser(TokenType.AS), - IdentifierNode.identifierParser(), + Parser.doAssert(Parser.terminalParser(TokenType.FROM), "'from as '"), + Parser.doAssert(StringLiteralNode._literalParser(), "source as a string"), + Parser.doAssert(Parser.terminalParser(TokenType.AS), "'as '"), + Parser.doAssert(IdentifierNode.identifierParser(), "variable name"), createNode=createNode ) @@ -71,6 +71,6 @@ class ImportNode(Node): return Parser.allOf( Parser.terminalParser(TokenType.IMPORT), - StringLiteralNode._literalParser(), + Parser.doAssert(StringLiteralNode._literalParser(), "source as a string"), createNode=createNode ) diff --git a/smnp/ast/node/integer.py b/smnp/ast/node/integer.py index ff2ca42..fccb86d 100644 --- a/smnp/ast/node/integer.py +++ b/smnp/ast/node/integer.py @@ -22,7 +22,7 @@ class IntegerLiteralNode(AccessNode): return Parser.allOf( Parser.terminalParser(TokenType.MINUS), - cls._positiveIntegerParser(), + Parser.doAssert(cls._positiveIntegerParser(), "integer"), createNode=createNode ) diff --git a/smnp/ast/node/invocation.py b/smnp/ast/node/invocation.py index 3b36f72..4dfe708 100644 --- a/smnp/ast/node/invocation.py +++ b/smnp/ast/node/invocation.py @@ -2,6 +2,7 @@ from smnp.ast.node.access import AccessNode from smnp.ast.node.expression import ExpressionNode from smnp.ast.node.iterable import abstractIterableParser from smnp.ast.node.model import Node +from smnp.ast.parser import Parser from smnp.token.type import TokenType @@ -9,7 +10,8 @@ class ArgumentsListNode(Node): @classmethod def _parse(cls, input): - return abstractIterableParser(ArgumentsListNode, TokenType.OPEN_PAREN, TokenType.CLOSE_PAREN, ExpressionNode.parse)(input) + return abstractIterableParser(ArgumentsListNode, TokenType.OPEN_PAREN, TokenType.CLOSE_PAREN, + Parser.doAssert(ExpressionNode.parse, "expression"))(input) class FunctionCall(AccessNode): diff --git a/smnp/ast/node/iterable.py b/smnp/ast/node/iterable.py index f816023..53fb7e6 100644 --- a/smnp/ast/node/iterable.py +++ b/smnp/ast/node/iterable.py @@ -41,7 +41,7 @@ def abstractIterableParser(iterableNodeType, openTokenType, closeTokenType, item return node return Parser.allOf( - Parser.terminalParser(TokenType.COMMA), + Parser.terminalParser(TokenType.COMMA, doAssert=True), itemParser, AbstractIterableTailNode.parse, createNode=createNode diff --git a/smnp/ast/node/list.py b/smnp/ast/node/list.py index f1194eb..0316918 100644 --- a/smnp/ast/node/list.py +++ b/smnp/ast/node/list.py @@ -1,6 +1,7 @@ from smnp.ast.node.access import AccessNode from smnp.ast.node.expression import ExpressionNode from smnp.ast.node.iterable import abstractIterableParser +from smnp.ast.parser import Parser from smnp.token.type import TokenType @@ -8,4 +9,5 @@ class ListNode(AccessNode): @classmethod def _literalParser(cls): - return abstractIterableParser(ListNode, TokenType.OPEN_SQUARE, TokenType.CLOSE_SQUARE, ExpressionNode.parse) + return abstractIterableParser(ListNode, TokenType.OPEN_SQUARE, TokenType.CLOSE_SQUARE, + Parser.doAssert(ExpressionNode.parse, "expression")) diff --git a/smnp/ast/node/program.py b/smnp/ast/node/program.py index 59879e8..a6991ac 100644 --- a/smnp/ast/node/program.py +++ b/smnp/ast/node/program.py @@ -21,7 +21,7 @@ class Program(Node): ExpressionNode.parse, ImportNode.parse, StatementNode.parse, - exception = SyntaxException(f"Unknown statement: {input.current().pos}") + exception = SyntaxException(f"Invalid statement: {input.currentToEndOfLine()}", input.current().pos) )(input) root = Program() diff --git a/smnp/ast/node/ret.py b/smnp/ast/node/ret.py index 17f1f79..d963287 100644 --- a/smnp/ast/node/ret.py +++ b/smnp/ast/node/ret.py @@ -27,6 +27,6 @@ class ReturnNode(StatementNode): return Parser.allOf( Parser.terminalParser(TokenType.RETURN), - ExpressionNode.parse, + Parser.doAssert(ExpressionNode.parse, "expression"), createNode=createNode )(input) \ No newline at end of file diff --git a/smnp/ast/node/variable.py b/smnp/ast/node/variable.py index 060e42b..8fc5ef6 100644 --- a/smnp/ast/node/variable.py +++ b/smnp/ast/node/variable.py @@ -36,7 +36,7 @@ class TypedVariableNode(ExpressionNode): return Parser.allOf( Parser.terminalParser(TokenType.TYPE, lambda val, pos: TypeNode.withValue(val, pos)), - IdentifierNode.identifierParser(), + Parser.doAssert(IdentifierNode.identifierParser(), "variable name"), createNode=createNode ) diff --git a/smnp/ast/parser.py b/smnp/ast/parser.py index a850f68..96198f3 100644 --- a/smnp/ast/parser.py +++ b/smnp/ast/parser.py @@ -1,5 +1,6 @@ from smnp.ast.node.ignore import IgnoredNode from smnp.ast.node.model import ParseResult, Node +from smnp.error.syntax import SyntaxException def parse(input): @@ -11,7 +12,7 @@ class Parser: # a -> A @staticmethod - def terminalParser(expectedType, createNode=None): + def terminalParser(expectedType, createNode=None, doAssert=False): def provideNode(value, pos): if createNode is None: return IgnoredNode(pos) @@ -22,6 +23,10 @@ class Parser: token = input.current() input.ahead() return ParseResult.OK(provideNode(token.value, token.pos)) + elif doAssert: + found = f", found '{input.current().rawValue}'" if input.hasCurrent() else "" + raise SyntaxException(f"Expected '{expectedType.key}'{found}", input.currentPos()) + return ParseResult.FAIL() return parse @@ -37,7 +42,11 @@ class Parser: return value if exception is not None: - raise exception + if callable(exception): + raise exception(input) + else: + raise exception + input.reset(snap) return ParseResult.FAIL() @@ -60,7 +69,10 @@ class Parser: if not result.result: if exception is not None: - raise exception + if callable(exception): + raise exception(input) + else: + raise exception input.reset(snap) return ParseResult.FAIL() @@ -114,3 +126,16 @@ class Parser: return parse + @staticmethod + def doAssert(parser, expected): + def parse(input): + result = parser(input) + + if not result.result: + found = f", found '{input.current().rawValue}'" if input.hasCurrent() else '' + + raise SyntaxException(f"Expected {expected}{found}", input.currentPos()) + + return result + + return parse diff --git a/smnp/ast/tools.py b/smnp/ast/tools.py index 215df23..fad3240 100644 --- a/smnp/ast/tools.py +++ b/smnp/ast/tools.py @@ -5,4 +5,4 @@ def assertToken(expected, input): if not input.hasCurrent(): raise SyntaxException(f"Expected '{expected}'") if expected != input.current().type: - raise SyntaxException(f"Expected '{expected}', found '{input.current().value}'", input.current().pos) + raise SyntaxException(f"Expected '{expected}', found '{input.current().rawValue}'", input.current().pos) diff --git a/smnp/main.py b/smnp/main.py index a519011..f16b958 100644 --- a/smnp/main.py +++ b/smnp/main.py @@ -15,9 +15,11 @@ def main(): tokens = tokenize(lines) ast = parse(tokens) + + ast.print() - + sys.exit(0) env = createEnvironment() evaluate(ast, env) diff --git a/smnp/runtime/evaluator.py b/smnp/runtime/evaluator.py index 40a93cb..9a724a1 100644 --- a/smnp/runtime/evaluator.py +++ b/smnp/runtime/evaluator.py @@ -1,57 +1,2 @@ -from smnp.ast.node.access import AccessNode -from smnp.ast.node.assignment import AssignmentNode -from smnp.ast.node.asterisk import AsteriskNode -from smnp.ast.node.block import BlockNode -from smnp.ast.node.colon import ColonNode -from smnp.ast.node.function import FunctionDefinitionNode, FunctionCallNode -from smnp.ast.node.identifier import IdentifierNode -from smnp.ast.node.integer import IntegerLiteralNode -from smnp.ast.node.list import ListNode -from smnp.ast.node.note import NoteLiteralNode -from smnp.ast.node.percent import PercentNode -from smnp.ast.node.program import Program -from smnp.ast.node.string import StringLiteralNode - def evaluate(input, environment): - from smnp.runtime.evaluators.access import evaluateAccess - from smnp.runtime.evaluators.assignment import evaluateAssignment - from smnp.runtime.evaluators.asterisk import evaluateAsterisk - from smnp.runtime.evaluators.block import evaluateBlock - from smnp.runtime.evaluators.colon import evaluateColon - from smnp.runtime.evaluators.function import evaluateFunctionDefinition, evaluateFunctionCall - from smnp.runtime.evaluators.identifier import evaluateIdentifier - from smnp.runtime.evaluators.integer import evaluateInteger - from smnp.runtime.evaluators.list import evaluateList - from smnp.runtime.evaluators.note import evaluateNote - from smnp.runtime.evaluators.percent import evaluatePercent - from smnp.runtime.evaluators.program import evaluateProgram - from smnp.runtime.evaluators.string import evaluateString - - if isinstance(input, Program): - return evaluateProgram(input, environment) - if isinstance(input, IntegerLiteralNode): - return evaluateInteger(input, environment) - if isinstance(input, PercentNode): - return evaluatePercent(input, environment) - if isinstance(input, StringLiteralNode): - return evaluateString(input, environment) - if isinstance(input, NoteLiteralNode): - return evaluateNote(input, environment) - if isinstance(input, FunctionDefinitionNode): - return evaluateFunctionDefinition(input, environment) - if isinstance(input, FunctionCallNode): - return evaluateFunctionCall(input, environment) - if isinstance(input, AccessNode): - return evaluateAccess(input, environment) - if isinstance(input, BlockNode): - return evaluateBlock(input, environment) - if isinstance(input, ListNode): - return evaluateList(input, environment) - if isinstance(input, AssignmentNode): - return evaluateAssignment(input, environment) - if isinstance(input, AsteriskNode): - return evaluateAsterisk(input, environment) - if isinstance(input, ColonNode): - return evaluateColon(input, environment) - if isinstance(input, IdentifierNode): - return evaluateIdentifier(input, environment) \ No newline at end of file + pass diff --git a/smnp/token/model.py b/smnp/token/model.py index 2e7b66b..08ee2e7 100644 --- a/smnp/token/model.py +++ b/smnp/token/model.py @@ -1,19 +1,25 @@ class Token: - def __init__(self, type, value, pos): + def __init__(self, type, value, pos, rawValue=None): self.type = type self.value = value self.pos = pos + if rawValue is None: + rawValue = value + self.rawValue = rawValue + def __str__(self): return "Token(" + str(self.type) + ", '" + str(self.value) + "', " + str(self.pos) + ")" + def __repr__(self): return self.__str__() class TokenList: - def __init__(self, tokens = []): + def __init__(self, tokens, lines): self.tokens = tokens self.cursor = 0 self.snap = 0 + self.lines = lines def append(self, token): self.tokens.append(token) @@ -26,6 +32,10 @@ class TokenList: raise RuntimeError(f"Cursor points to not existing token! Cursor = {self.cursor}, len = {len(self.tokens)}") return self.tokens[self.cursor] + def currentPos(self): + #TODO maybe change raw pos (position) tuple to some class with method "nextCol()", "nextRow()" etc. + return self.current().pos if self.hasCurrent() else (self.tokens[-1].pos[0], self.tokens[-1].pos[1]+1) if len(self.tokens) > 0 else None + def isCurrent(self, type): return self.hasCurrent() and self.current().type == type @@ -49,7 +59,10 @@ class TokenList: def reset(self, snap): self.cursor = snap - + + def currentToEndOfLine(self): + return self.lines[self.current().pos[0]][self.current().pos[1]:] + def __str__(self): return f"[Current({self.cursor}): {self.current() if self.hasCurrent() else 'out of tokens'}\n{', '.join([str(token) for token in self.tokens])}]" diff --git a/smnp/token/tokenizer.py b/smnp/token/tokenizer.py index 622f28e..4540ba3 100644 --- a/smnp/token/tokenizer.py +++ b/smnp/token/tokenizer.py @@ -70,7 +70,7 @@ def tokenize(lines): current += consumedChars tokens.append(token) - return TokenList(filterTokens(filters, tokens)) + return TokenList(filterTokens(filters, tokens), lines) def combinedTokenizer(line, current, lineNumber): diff --git a/smnp/token/tokenizers/note.py b/smnp/token/tokenizers/note.py index b578b44..cc2be13 100644 --- a/smnp/token/tokenizers/note.py +++ b/smnp/token/tokenizers/note.py @@ -11,35 +11,43 @@ def tokenizeNote(input, current, line): octave = None duration = None dot = False + rawValue = '' if input[current] == '@': + rawValue += input[current+consumedChars] consumedChars += 1 if input[current+consumedChars] in ('C', 'c', 'D', 'd', 'E', 'e', 'F', 'f', 'G', 'g', 'A', 'a', 'H', 'h', 'B', 'b'): + rawValue += input[current + consumedChars] notePitch = input[current+consumedChars] consumedChars += 1 - if current+consumedChars < len(input) and input[current+consumedChars] in ('b', '#'): + if current+consumedChars < len(input) and input[current+consumedChars] in ('b', '#'): + rawValue += input[current + consumedChars] notePitch += input[current+consumedChars] consumedChars += 1 - if current+consumedChars < len(input) and re.match(r'\d', input[current+consumedChars]): + if current+consumedChars < len(input) and re.match(r'\d', input[current+consumedChars]): + rawValue += input[current + consumedChars] octave = input[current+consumedChars] consumedChars += 1 if current+consumedChars < len(input) and input[current+consumedChars] == ':': + rawValue += input[current + consumedChars] duration = '' consumedChars += 1 while current+consumedChars < len(input) and re.match(r'\d', input[current+consumedChars]): + rawValue += input[current + consumedChars] duration += input[current+consumedChars] consumedChars += 1 if len(duration) == 0: return (0, None) dot = (current+consumedChars) < len(input) and input[current+consumedChars] == 'd' if dot: + rawValue += input[current + consumedChars] consumedChars += 1 octave = int(octave) if octave is not None else None duration = int(duration) if duration is not None else None value = Note(notePitch, octave, duration, dot) - return (consumedChars, Token(TokenType.NOTE, value, (line, current))) + return (consumedChars, Token(TokenType.NOTE, value, (line, current), rawValue)) return (0, None) diff --git a/smnp/token/type.py b/smnp/token/type.py index 67cf9d4..bf0b422 100644 --- a/smnp/token/type.py +++ b/smnp/token/type.py @@ -1,28 +1,36 @@ -from enum import Enum, auto +from enum import Enum class TokenType(Enum): - OPEN_PAREN = auto() - CLOSE_PAREN = auto() - ASTERISK = auto() - STRING = auto() - IDENTIFIER = auto() - COMMA = auto() - INTEGER = auto() - OPEN_BRACKET = auto() - CLOSE_BRACKET = auto() - ASSIGN = auto() - NOTE = auto() - COMMENT = auto() - PERCENT = auto() - MINUS = auto() - FUNCTION = auto() - RETURN = auto() - DOT = auto() - OPEN_SQUARE = auto() - CLOSE_SQUARE = auto() - TYPE = auto() - EXTEND = auto() - IMPORT = auto() - FROM = auto() - AS = auto() \ No newline at end of file + OPEN_PAREN = '(' + CLOSE_PAREN = ')' + ASTERISK = '*' + STRING = 'string' + IDENTIFIER = 'identifier' + COMMA = ',' + INTEGER = 'integer' + OPEN_BRACKET = '{' + CLOSE_BRACKET = '}' + ASSIGN = '=' + NOTE = 'note' + COMMENT = 'comment' + PERCENT = 'percent' + MINUS = '-' + FUNCTION = 'function' + RETURN = 'return' + DOT = '.' + OPEN_SQUARE = '[' + CLOSE_SQUARE = ']' + TYPE = 'type' + EXTEND = 'extend' + IMPORT = 'import' + FROM = 'from' + AS = 'as' + + @property + def key(self): + return self.value + + @key.setter + def key(self, value): + raise RuntimeError("Cannot change key of token type")