bartek/smnp-py
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve errors raising by parser

Browse Source
This commit is contained in:
Bartłomiej Pluta
2019-07-06 18:04:05 +02:00
parent 6fd49ba54a
commit fbb3f79731
22 changed files with 125 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
smnp/ast/node/access.py
View File

@@ -1,6 +1,7 @@
from smnp.ast.node.expression import ExpressionNode
from smnp.ast.node.ignore import IgnoredNode
from smnp.ast.parser import Parser
from smnp.error.syntax import SyntaxException
from smnp.token.type import TokenType
@@ -50,5 +51,6 @@ class AccessNode(ExpressionNode):
return Parser.oneOf(
IdentifierNode._literalParser(),
IdentifierNode._functionCallParser()
IdentifierNode._functionCallParser(),
exception=lambda input: SyntaxException(f"Expected property name or method call, found '{input.current().rawValue}'", input.currentPos())
)

4
smnp/ast/node/block.py
View File

@@ -14,7 +14,7 @@ class BlockNode(StatementNode):
return Parser.loop(
Parser.terminalParser(TokenType.OPEN_BRACKET),
StatementNode.parse,
Parser.doAssert(StatementNode.parse, f"statement or '{TokenType.CLOSE_BRACKET.key}'"),
Parser.terminalParser(TokenType.CLOSE_BRACKET),
createNode=createNode
createNode=createNode,
)(input)

2
smnp/ast/node/expression.py
View File

@@ -44,7 +44,7 @@ class ExpressionNode(Node):
return Parser.allOf(
cls._expressionParser(),
Parser.terminalParser(TokenType.ASTERISK),
StatementNode.parse,
Parser.doAssert(StatementNode.parse, 'statement'),
createNode=createNode
)

10
smnp/ast/node/extend.py
View File

@@ -48,10 +48,10 @@ class ExtendNode(StatementNode):
return Parser.allOf(
Parser.terminalParser(TokenType.EXTEND),
TypeNode.parse,
Parser.terminalParser(TokenType.AS),
IdentifierNode.identifierParser(),
cls._methodsDeclarationsParser(),
Parser.doAssert(TypeNode.parse, "type being extended"),
Parser.terminalParser(TokenType.AS, doAssert=True),
Parser.doAssert(IdentifierNode.identifierParser(), "variable name"),
Parser.doAssert(cls._methodsDeclarationsParser(), "methods declarations"),
createNode=createNode
)(input)
@@ -64,7 +64,7 @@ class ExtendNode(StatementNode):
return Parser.loop(
Parser.terminalParser(TokenType.OPEN_BRACKET),
FunctionDefinitionNode.parse,
Parser.doAssert(FunctionDefinitionNode.parse, f"method declaration or '{TokenType.CLOSE_BRACKET.key}'"),
Parser.terminalParser(TokenType.CLOSE_BRACKET),
createNode=createNode
)

6
smnp/ast/node/function.py
View File

@@ -56,9 +56,9 @@ class FunctionDefinitionNode(StatementNode):
return Parser.allOf(
Parser.terminalParser(TokenType.FUNCTION),
IdentifierNode.identifierParser(),
cls._argumentsDeclarationParser(),
BlockNode.parse,
Parser.doAssert(IdentifierNode.identifierParser(), "function name"),
Parser.doAssert(cls._argumentsDeclarationParser(), "arguments list"),
Parser.doAssert(BlockNode.parse, "function body"),
createNode=createNode
)(input)

2
smnp/ast/node/identifier.py
View File

@@ -30,7 +30,7 @@ class IdentifierNode(AccessNode):
return Parser.allOf(
IdentifierNode.identifierParser(),
Parser.terminalParser(TokenType.ASSIGN),
ExpressionNode.parse,
Parser.doAssert(ExpressionNode.parse, "expression"),
createNode=createNode
)

10
smnp/ast/node/imports.py
View File

@@ -55,10 +55,10 @@ class ImportNode(Node):
return Parser.allOf(
Parser.terminalParser(TokenType.IMPORT),
TypeNode.parse,
Parser.terminalParser(TokenType.FROM),
StringLiteralNode._literalParser(),
Parser.terminalParser(TokenType.AS),
IdentifierNode.identifierParser(),
Parser.doAssert(Parser.terminalParser(TokenType.FROM), "'from <source> as <variable name>'"),
Parser.doAssert(StringLiteralNode._literalParser(), "source as a string"),
Parser.doAssert(Parser.terminalParser(TokenType.AS), "'as <variable name>'"),
Parser.doAssert(IdentifierNode.identifierParser(), "variable name"),
createNode=createNode
)
@@ -71,6 +71,6 @@ class ImportNode(Node):
return Parser.allOf(
Parser.terminalParser(TokenType.IMPORT),
StringLiteralNode._literalParser(),
Parser.doAssert(StringLiteralNode._literalParser(), "source as a string"),
createNode=createNode
)

2
smnp/ast/node/integer.py
View File

@@ -22,7 +22,7 @@ class IntegerLiteralNode(AccessNode):
return Parser.allOf(
Parser.terminalParser(TokenType.MINUS),
cls._positiveIntegerParser(),
Parser.doAssert(cls._positiveIntegerParser(), "integer"),
createNode=createNode
)

4
smnp/ast/node/invocation.py
View File

@@ -2,6 +2,7 @@ from smnp.ast.node.access import AccessNode
from smnp.ast.node.expression import ExpressionNode
from smnp.ast.node.iterable import abstractIterableParser
from smnp.ast.node.model import Node
from smnp.ast.parser import Parser
from smnp.token.type import TokenType
@@ -9,7 +10,8 @@ class ArgumentsListNode(Node):
@classmethod
def _parse(cls, input):
return abstractIterableParser(ArgumentsListNode, TokenType.OPEN_PAREN, TokenType.CLOSE_PAREN, ExpressionNode.parse)(input)
return abstractIterableParser(ArgumentsListNode, TokenType.OPEN_PAREN, TokenType.CLOSE_PAREN,
Parser.doAssert(ExpressionNode.parse, "expression"))(input)
class FunctionCall(AccessNode):

2
smnp/ast/node/iterable.py
View File

@@ -41,7 +41,7 @@ def abstractIterableParser(iterableNodeType, openTokenType, closeTokenType, item
return node
return Parser.allOf(
Parser.terminalParser(TokenType.COMMA),
Parser.terminalParser(TokenType.COMMA, doAssert=True),
itemParser,
AbstractIterableTailNode.parse,
createNode=createNode

4
smnp/ast/node/list.py
View File

@@ -1,6 +1,7 @@
from smnp.ast.node.access import AccessNode
from smnp.ast.node.expression import ExpressionNode
from smnp.ast.node.iterable import abstractIterableParser
from smnp.ast.parser import Parser
from smnp.token.type import TokenType
@@ -8,4 +9,5 @@ class ListNode(AccessNode):
@classmethod
def _literalParser(cls):
return abstractIterableParser(ListNode, TokenType.OPEN_SQUARE, TokenType.CLOSE_SQUARE, ExpressionNode.parse)
return abstractIterableParser(ListNode, TokenType.OPEN_SQUARE, TokenType.CLOSE_SQUARE,
Parser.doAssert(ExpressionNode.parse, "expression"))

2
smnp/ast/node/program.py
View File

@@ -21,7 +21,7 @@ class Program(Node):
ExpressionNode.parse,
ImportNode.parse,
StatementNode.parse,
exception = SyntaxException(f"Unknown statement: {input.current().pos}")
exception = SyntaxException(f"Invalid statement: {input.currentToEndOfLine()}", input.current().pos)
)(input)
root = Program()

2
smnp/ast/node/ret.py
View File

@@ -27,6 +27,6 @@ class ReturnNode(StatementNode):
return Parser.allOf(
Parser.terminalParser(TokenType.RETURN),
ExpressionNode.parse,
Parser.doAssert(ExpressionNode.parse, "expression"),
createNode=createNode
)(input)

2
smnp/ast/node/variable.py
View File

@@ -36,7 +36,7 @@ class TypedVariableNode(ExpressionNode):
return Parser.allOf(
Parser.terminalParser(TokenType.TYPE, lambda val, pos: TypeNode.withValue(val, pos)),
IdentifierNode.identifierParser(),
Parser.doAssert(IdentifierNode.identifierParser(), "variable name"),
createNode=createNode
)

27
smnp/ast/parser.py
View File

@@ -1,5 +1,6 @@
from smnp.ast.node.ignore import IgnoredNode
from smnp.ast.node.model import ParseResult, Node
from smnp.error.syntax import SyntaxException
def parse(input):
@@ -11,7 +12,7 @@ class Parser:
# a -> A
@staticmethod
def terminalParser(expectedType, createNode=None):
def terminalParser(expectedType, createNode=None, doAssert=False):
def provideNode(value, pos):
if createNode is None:
return IgnoredNode(pos)
@@ -22,6 +23,10 @@ class Parser:
token = input.current()
input.ahead()
return ParseResult.OK(provideNode(token.value, token.pos))
elif doAssert:
found = f", found '{input.current().rawValue}'" if input.hasCurrent() else ""
raise SyntaxException(f"Expected '{expectedType.key}'{found}", input.currentPos())
return ParseResult.FAIL()
return parse
@@ -37,8 +42,12 @@ class Parser:
return value
if exception is not None:
if callable(exception):
raise exception(input)
else:
raise exception
input.reset(snap)
return ParseResult.FAIL()
@@ -60,6 +69,9 @@ class Parser:
if not result.result:
if exception is not None:
if callable(exception):
raise exception(input)
else:
raise exception
input.reset(snap)
@@ -114,3 +126,16 @@ class Parser:
return parse
@staticmethod
def doAssert(parser, expected):
def parse(input):
result = parser(input)
if not result.result:
found = f", found '{input.current().rawValue}'" if input.hasCurrent() else ''
raise SyntaxException(f"Expected {expected}{found}", input.currentPos())
return result
return parse

2
smnp/ast/tools.py
View File

@@ -5,4 +5,4 @@ def assertToken(expected, input):
if not input.hasCurrent():
raise SyntaxException(f"Expected '{expected}'")
if expected != input.current().type:
raise SyntaxException(f"Expected '{expected}', found '{input.current().value}'", input.current().pos)
raise SyntaxException(f"Expected '{expected}', found '{input.current().rawValue}'", input.current().pos)

4
smnp/main.py
View File

@@ -15,9 +15,11 @@ def main():
tokens = tokenize(lines)
ast = parse(tokens)
ast.print()
sys.exit(0)
env = createEnvironment()
evaluate(ast, env)

57
smnp/runtime/evaluator.py
View File

@@ -1,57 +1,2 @@
from smnp.ast.node.access import AccessNode
from smnp.ast.node.assignment import AssignmentNode
from smnp.ast.node.asterisk import AsteriskNode
from smnp.ast.node.block import BlockNode
from smnp.ast.node.colon import ColonNode
from smnp.ast.node.function import FunctionDefinitionNode, FunctionCallNode
from smnp.ast.node.identifier import IdentifierNode
from smnp.ast.node.integer import IntegerLiteralNode
from smnp.ast.node.list import ListNode
from smnp.ast.node.note import NoteLiteralNode
from smnp.ast.node.percent import PercentNode
from smnp.ast.node.program import Program
from smnp.ast.node.string import StringLiteralNode
def evaluate(input, environment):
from smnp.runtime.evaluators.access import evaluateAccess
from smnp.runtime.evaluators.assignment import evaluateAssignment
from smnp.runtime.evaluators.asterisk import evaluateAsterisk
from smnp.runtime.evaluators.block import evaluateBlock
from smnp.runtime.evaluators.colon import evaluateColon
from smnp.runtime.evaluators.function import evaluateFunctionDefinition, evaluateFunctionCall
from smnp.runtime.evaluators.identifier import evaluateIdentifier
from smnp.runtime.evaluators.integer import evaluateInteger
from smnp.runtime.evaluators.list import evaluateList
from smnp.runtime.evaluators.note import evaluateNote
from smnp.runtime.evaluators.percent import evaluatePercent
from smnp.runtime.evaluators.program import evaluateProgram
from smnp.runtime.evaluators.string import evaluateString
if isinstance(input, Program):
return evaluateProgram(input, environment)
if isinstance(input, IntegerLiteralNode):
return evaluateInteger(input, environment)
if isinstance(input, PercentNode):
return evaluatePercent(input, environment)
if isinstance(input, StringLiteralNode):
return evaluateString(input, environment)
if isinstance(input, NoteLiteralNode):
return evaluateNote(input, environment)
if isinstance(input, FunctionDefinitionNode):
return evaluateFunctionDefinition(input, environment)
if isinstance(input, FunctionCallNode):
return evaluateFunctionCall(input, environment)
if isinstance(input, AccessNode):
return evaluateAccess(input, environment)
if isinstance(input, BlockNode):
return evaluateBlock(input, environment)
if isinstance(input, ListNode):
return evaluateList(input, environment)
if isinstance(input, AssignmentNode):
return evaluateAssignment(input, environment)
if isinstance(input, AsteriskNode):
return evaluateAsterisk(input, environment)
if isinstance(input, ColonNode):
return evaluateColon(input, environment)
if isinstance(input, IdentifierNode):
return evaluateIdentifier(input, environment)
pass

17
smnp/token/model.py
View File

@@ -1,19 +1,25 @@
class Token:
def __init__(self, type, value, pos):
def __init__(self, type, value, pos, rawValue=None):
self.type = type
self.value = value
self.pos = pos
if rawValue is None:
rawValue = value
self.rawValue = rawValue
def __str__(self):
return "Token(" + str(self.type) + ", '" + str(self.value) + "', " + str(self.pos) + ")"
def __repr__(self):
return self.__str__()
class TokenList:
def __init__(self, tokens = []):
def __init__(self, tokens, lines):
self.tokens = tokens
self.cursor = 0
self.snap = 0
self.lines = lines
def append(self, token):
self.tokens.append(token)
@@ -26,6 +32,10 @@ class TokenList:
raise RuntimeError(f"Cursor points to not existing token! Cursor = {self.cursor}, len = {len(self.tokens)}")
return self.tokens[self.cursor]
def currentPos(self):
#TODO maybe change raw pos (position) tuple to some class with method "nextCol()", "nextRow()" etc.
return self.current().pos if self.hasCurrent() else (self.tokens[-1].pos[0], self.tokens[-1].pos[1]+1) if len(self.tokens) > 0 else None
def isCurrent(self, type):
return self.hasCurrent() and self.current().type == type
@@ -50,6 +60,9 @@ class TokenList:
def reset(self, snap):
self.cursor = snap
def currentToEndOfLine(self):
return self.lines[self.current().pos[0]][self.current().pos[1]:]
def __str__(self):
return f"[Current({self.cursor}): {self.current() if self.hasCurrent() else 'out of tokens'}\n{', '.join([str(token) for token in self.tokens])}]"

2
smnp/token/tokenizer.py
View File

@@ -70,7 +70,7 @@ def tokenize(lines):
current += consumedChars
tokens.append(token)
return TokenList(filterTokens(filters, tokens))
return TokenList(filterTokens(filters, tokens), lines)
def combinedTokenizer(line, current, lineNumber):

10
smnp/token/tokenizers/note.py
View File

@@ -11,35 +11,43 @@ def tokenizeNote(input, current, line):
octave = None
duration = None
dot = False
rawValue = ''
if input[current] == '@':
rawValue += input[current+consumedChars]
consumedChars += 1
if input[current+consumedChars] in ('C', 'c', 'D', 'd', 'E', 'e', 'F', 'f', 'G', 'g', 'A', 'a', 'H', 'h', 'B', 'b'):
rawValue += input[current + consumedChars]
notePitch = input[current+consumedChars]
consumedChars += 1
if current+consumedChars < len(input) and input[current+consumedChars] in ('b', '#'):
rawValue += input[current + consumedChars]
notePitch += input[current+consumedChars]
consumedChars += 1
if current+consumedChars < len(input) and re.match(r'\d', input[current+consumedChars]):
rawValue += input[current + consumedChars]
octave = input[current+consumedChars]
consumedChars += 1
if current+consumedChars < len(input) and input[current+consumedChars] == ':':
rawValue += input[current + consumedChars]
duration = ''
consumedChars += 1
while current+consumedChars < len(input) and re.match(r'\d', input[current+consumedChars]):
rawValue += input[current + consumedChars]
duration += input[current+consumedChars]
consumedChars += 1
if len(duration) == 0:
return (0, None)
dot = (current+consumedChars) < len(input) and input[current+consumedChars] == 'd'
if dot:
rawValue += input[current + consumedChars]
consumedChars += 1
octave = int(octave) if octave is not None else None
duration = int(duration) if duration is not None else None
value = Note(notePitch, octave, duration, dot)
return (consumedChars, Token(TokenType.NOTE, value, (line, current)))
return (consumedChars, Token(TokenType.NOTE, value, (line, current), rawValue))
return (0, None)

58
smnp/token/type.py
View File

@@ -1,28 +1,36 @@
from enum import Enum, auto
from enum import Enum
class TokenType(Enum):
OPEN_PAREN = auto()
CLOSE_PAREN = auto()
ASTERISK = auto()
STRING = auto()
IDENTIFIER = auto()
COMMA = auto()
INTEGER = auto()
OPEN_BRACKET = auto()
CLOSE_BRACKET = auto()
ASSIGN = auto()
NOTE = auto()
COMMENT = auto()
PERCENT = auto()
MINUS = auto()
FUNCTION = auto()
RETURN = auto()
DOT = auto()
OPEN_SQUARE = auto()
CLOSE_SQUARE = auto()
TYPE = auto()
EXTEND = auto()
IMPORT = auto()
FROM = auto()
AS = auto()
OPEN_PAREN = '('
CLOSE_PAREN = ')'
ASTERISK = '*'
STRING = 'string'
IDENTIFIER = 'identifier'
COMMA = ','
INTEGER = 'integer'
OPEN_BRACKET = '{'
CLOSE_BRACKET = '}'
ASSIGN = '='
NOTE = 'note'
COMMENT = 'comment'
PERCENT = 'percent'
MINUS = '-'
FUNCTION = 'function'
RETURN = 'return'
DOT = '.'
OPEN_SQUARE = '['
CLOSE_SQUARE = ']'
TYPE = 'type'
EXTEND = 'extend'
IMPORT = 'import'
FROM = 'from'
AS = 'as'
@property
def key(self):
return self.value
@key.setter
def key(self, value):
raise RuntimeError("Cannot change key of token type")