Create new parser (works for lists so far)

This commit is contained in:
Bartłomiej Pluta
2019-07-05 16:45:59 +02:00
parent 23e0f3f33e
commit ed73aa1ad1
32 changed files with 516 additions and 59 deletions

42
grammar Normal file
View File

@@ -0,0 +1,42 @@
integer := ...
string := ...
note := ...
identifier := ...
expr := integer
expr := string
expr := note
expr := identifier
expr := access
expr := assignment
expr := functionCall
# left associative
access := expr '.' expr
# right associative
asterisk := expr '*' stmt
stmt := asterisk
stmt := block
stmt := return
stmt := functionDefinition
# right associative
assignment := identifier '=' expr
list := '(' ')'
list := '(' expr listTail
listTail := expr ', ' listTail
listTail := ')'
percent := integer '%'
return := 'return' expr
block := '{' stmt* '}'
functionCall := identifier list
functionDefinition := 'function' identifier list block

42
reduced-grammar Normal file
View File

@@ -0,0 +1,42 @@
# Tokenizer
DIGIT = [0-9]
ID = [a-zA-Z_]
CHAR = ... \ '"'
PITCH = 'c' | 'd' | 'e' | 'f' | 'g' | 'a' | 'h'
PITCH_MODIFIER = 'b' | '#'
integer := '-' DIGIT+ | DIGIT+
string := '"' CHAR* '"'
note := '@' PITCH PITCH_MODIFIER? DIGIT? ['.' DIGIT+ 'd'?]?
identifier := ID [ID|DIGIT]*
percent := DIGIT+ '%'
# Parser
expr := integer accessTail | integer
expr := percent accessTail | percent
expr := string accessTail | string
expr := note accessTail | note
expr := identifier accessTail | identifier '=' expr | functionCall | identifier
expr := list accessTail | list
expr := functionCall accessTail | functionCall
functionCall := identifier list
accessTail := '.' expr accessTail | e
list := '[' ']' | '[' expr listTail
listTail := expr ', ' listTail | ']'
argList := '(' ')' | '(' expr argListTail
argListTail := expr ', ' argListTail | ')'
block := '{' stmt* '}'
stmt := expr asteriskTail | expr #nie wiem czy zamiast 'expr asteriskTail' nie powinno być wprost co może wyprodukować iterator dla asterisk
asteriskTail := '*' stmt | e
stmt := block
stmt := 'return' expr
stmt := 'function' identifier list block
program := stmt*

View File

@@ -8,7 +8,7 @@ from smnp.token.type import TokenType
# i potem sprawdzać przy wszystkich parent.pop(-1) czy pobrany z parenta element
# jest rzeczywiście wyrażeniem, bo teraz możliwe jest np. {}.fun()
def parseAccess(input, parent):
if input.current().type == TokenType.DOT:
if input.isCurrent(TokenType.DOT):
token = input.current()
input.ahead()

View File

@@ -1,2 +0,0 @@

View File

@@ -5,7 +5,7 @@ from smnp.token.type import TokenType
# asterisk -> expr '*' stmt
def parseAsterisk(expr, input, parent):
if input.hasMore() and input.current().type == TokenType.ASTERISK:
if input.hasMore() and input.isCurrent(TokenType.ASTERISK):
token = input.current()
input.ahead()

View File

@@ -12,7 +12,7 @@ def parseBlock(input, parent):
node = BlockNode(parent, token.pos)
# '}'
if input.hasCurrent() and input.current().type == TokenType.CLOSE_BRACKET:
if input.isCurrent(TokenType.CLOSE_BRACKET):
input.ahead()
return node
@@ -28,7 +28,7 @@ def parseBlock(input, parent):
# blockItem -> stmt | '}'
def parseBlockItem(input, parent):
# '}'
if input.hasCurrent() and input.current().type == TokenType.CLOSE_BRACKET:
if input.isCurrent(TokenType.CLOSE_BRACKET):
close = CloseBlockNode(parent, input.current().pos)
input.ahead()
return close

View File

@@ -6,7 +6,7 @@ from smnp.token.type import TokenType
# colon -> expr ':' expr
def parseColon(expr1, input, parent):
if input.hasCurrent() and input.current().type == TokenType.COLON:
if input.isCurrent(TokenType.COLON):
token = input.current()
input.ahead()
expr2 = parseExpression(input, parent)

View File

@@ -7,7 +7,7 @@ from smnp.token.type import TokenType
def parseFunctionDefinition(input, parent):
if input.current().type == TokenType.FUNCTION:
if input.isCurrent(TokenType.FUNCTION):
token = input.current()
input.ahead()

View File

@@ -3,12 +3,13 @@ from smnp.ast.node.function import FunctionCallNode
from smnp.ast.node.identifier import IdentifierNode
from smnp.ast.parsers.expression import parseExpression
from smnp.ast.parsers.list import parseList
from smnp.ast.tools import greedy
from smnp.token.type import TokenType
# id -> IDENTIFIER
def parseIdentifier(input, parent):
if input.current().type == TokenType.IDENTIFIER:
if input.isCurrent(TokenType.IDENTIFIER):
identifier = IdentifierNode(input.current().value, parent, input.current().pos)
input.ahead()
@@ -28,7 +29,7 @@ def parseIdentifierOrFunctionCallOrAssignment(input, parent):
token = input.current()
input.ahead()
expr = parseExpression(input, parent)
expr = greedy(parseExpression)(input, parent)
assignment = AssignmentNode(identifier, expr, parent, token.pos)
identifier.parent = assignment

View File

@@ -5,7 +5,7 @@ from smnp.token.type import TokenType
# int -> INTEGER
def parseInteger(input, parent):
if input.current().type == TokenType.INTEGER:
if input.isCurrent(TokenType.INTEGER):
integer = IntegerLiteralNode(int(input.current().value), parent, input.current().pos)
input.ahead()
@@ -17,7 +17,7 @@ def parseInteger(input, parent):
# int -> int
def parseIntegerAndPercent(input, parent):
integer = parseInteger(input, parent)
if integer is not None and input.hasCurrent() and input.current().type == TokenType.PERCENT:
if integer is not None and input.isCurrent(TokenType.PERCENT):
percent = PercentNode(integer, parent, input.current().pos)
integer.parent = percent
input.ahead()

View File

@@ -1,17 +1,17 @@
from smnp.ast.node.list import ListNode, ListItemNode, CloseListNode
from smnp.ast.parsers.expression import parseExpression
from smnp.ast.tools import rollup, assertToken
from smnp.ast.tools import greedy, assertToken
from smnp.token.type import TokenType
# list -> CLOSE_PAREN | expr listTail
def parseList(input, parent):
if input.current().type == TokenType.OPEN_PAREN:
if input.isCurrent(TokenType.OPEN_PAREN):
node = ListNode(parent, input.current().pos)
input.ahead()
# list -> CLOSE_PAREN (end of list)
if input.hasCurrent() and input.current().type == TokenType.CLOSE_PAREN:
if input.isCurrent(TokenType.CLOSE_PAREN):
close = CloseListNode(node, input.current().pos)
node.append(close)
input.ahead()
@@ -20,7 +20,7 @@ def parseList(input, parent):
# list -> expr listTail
if input.hasCurrent():
token = input.current()
expr = parseExpression(input, node)
expr = greedy(parseExpression)(input, parent)
item = ListItemNode(expr, node, token.pos)
expr.parent = item
node.append(item)
@@ -33,7 +33,7 @@ def parseList(input, parent):
# listTail -> COMMA expr listTail | CLOSE_PAREN
def parseListTail(input, parent):
# listTail -> CLOSE_PAREN
if input.hasCurrent() and input.current().type == TokenType.CLOSE_PAREN:
if input.isCurrent(TokenType.CLOSE_PAREN):
close = CloseListNode(parent, input.current().pos)
input.ahead()
return close
@@ -42,7 +42,7 @@ def parseListTail(input, parent):
if input.hasCurrent() and input.hasMore():
assertToken(TokenType.COMMA, input)
input.ahead()
expr = rollup(parseExpression)(input, parent)
expr = greedy(parseExpression)(input, parent)
if expr is not None:
item = ListItemNode(expr, parent, expr.pos)
expr.parent = item

View File

@@ -5,10 +5,13 @@ from smnp.token.type import TokenType
# minus -> '-' int
def parseMinus(input, parent):
if input.current().type == TokenType.MINUS:
if input.isCurrent(TokenType.MINUS):
token = input.current()
input.ahead()
expr = parseInteger(input, parent)
if input.hasCurrent():
expr = parseInteger(input, parent)
return IntegerLiteralNode(-expr.value, parent, token.pos)
return IntegerLiteralNode(-expr.value, parent, token.pos)
return None

View File

@@ -7,7 +7,7 @@ from smnp.token.type import TokenType
# note -> NOTE
def parseNote(input, parent):
if input.current().type == TokenType.NOTE:
if input.isCurrent(TokenType.NOTE):
token = input.current()
value = token.value
consumedChars = 1

View File

@@ -4,7 +4,7 @@ from smnp.token.type import TokenType
def parseReturn(input, parent):
if input.current().type == TokenType.RETURN:
if input.isCurrent(TokenType.RETURN):
token = input.current()
input.ahead()

View File

@@ -4,7 +4,7 @@ from smnp.token.type import TokenType
# string -> STRING
def parseString(input, parent):
if input.current().type == TokenType.STRING:
if input.isCurrent(TokenType.STRING):
string = StringLiteralNode(input.current().value[1:len(input.current().value) - 1], parent, input.current().pos)
input.ahead()

View File

@@ -2,7 +2,7 @@ from smnp.ast.node.model import Node
from smnp.error.syntax import SyntaxException
def rollup(parser):
def greedy(parser):
def _rollup(input, parent):
node = Node(None, (-1, -1))
elem = parser(input, node)

View File

@@ -13,7 +13,7 @@ class Environment():
def invokeMethod(self, name, object, args):
for method in self.methods: # TODO to działa tylko dla wbudowanych funkcji
if method.name == name:
ret = method.call(self, [object, *args])
ret = method.call(self, [object, *args.value])
if ret is not None:
return ret
raise MethodNotFoundException(object.type, name) # TODO method not found

View File

@@ -1,8 +1,8 @@
import sys
from smnp.ast.parser import parse
from smnp.environment.factory import createEnvironment
from smnp.error.base import SmnpException
from smnp.newast.node.program import Program
from smnp.runtime.evaluator import evaluate
from smnp.token.tokenizer import tokenize
@@ -14,8 +14,10 @@ def main():
tokens = tokenize(lines)
ast = parse(tokens)
ast = Program.parse(tokens)
ast.node.print()
sys.exit(0)
env = createEnvironment()
evaluate(ast, env)

0
smnp/newast/__init__.py Normal file
View File

View File

View File

@@ -0,0 +1,37 @@
from smnp.newast.node.model import Node
from smnp.newast.node.none import NoneNode
from smnp.newast.parser import Parser
class ExpressionNode(Node):
def __init__(self, pos):
super().__init__(pos, [NoneNode()])
@property
def value(self):
return self[0]
@value.setter
def value(self, v):
self[0] = v
@classmethod
def withValue(cls, pos, v):
node = cls(pos)
node.value = v
return node
@classmethod
def _parse(cls, input):
from smnp.newast.node.integer import IntegerLiteralNode
from smnp.newast.node.string import StringLiteralNode
from smnp.newast.node.list import ListNode
return Parser.oneOf(
IntegerLiteralNode.parse,
StringLiteralNode.parse,
ListNode.parse
)(input)

View File

@@ -0,0 +1,6 @@
from smnp.newast.node.model import Node
class IgnoredNode(Node):
def __init__(self, pos):
super().__init__(pos)

View File

@@ -0,0 +1,11 @@
from smnp.newast.node.expression import ExpressionNode
from smnp.newast.parser import Parser
from smnp.token.type import TokenType
class IntegerLiteralNode(ExpressionNode):
@classmethod
def _parse(cls, input):
createNode = lambda v, pos: IntegerLiteralNode.withValue(pos, v)
return Parser.terminalParser(TokenType.INTEGER, createNode)(input)

102
smnp/newast/node/list.py Normal file
View File

@@ -0,0 +1,102 @@
from smnp.newast.node.expression import ExpressionNode
from smnp.newast.node.model import Node
from smnp.newast.node.none import NoneNode
from smnp.newast.parser import Parser
from smnp.token.type import TokenType
class ListTailNode(ExpressionNode):
def __init__(self, pos):
super().__init__(pos)
self.children.append(NoneNode())
@property
def next(self):
return self[1]
@next.setter
def next(self, value):
self[1] = value
@classmethod
def _parse(cls, input):
return Parser.oneOf(
ListTailNode._parser1(),
ListTailNode._parser2(),
)(input)
# listTail := ']'
@staticmethod
def _parser1():
return Parser.terminalParser(TokenType.CLOSE_PAREN)
# listTail := ',' expr listTail
@staticmethod
def _parser2():
def createNode(comma, expr, listTail):
node = ListTailNode(expr.pos)
node.value = expr
node.next = listTail
return node
return Parser.allOf(
Parser.terminalParser(TokenType.COMMA),
ExpressionNode.parse,
ListTailNode.parse,
createNode=createNode
)
class ListNode(ExpressionNode):
def __init__(self, pos):
super().__init__(pos)
self.children.append(NoneNode())
@property
def next(self):
return self[1]
@next.setter
def next(self, value):
self[1] = value
@classmethod
def _parse(cls, input):
return Parser.oneOf(
ListNode._parser1(),
ListNode._parser2()
)(input)
# list := '[' ']'
@staticmethod
def _parser1():
def emptyList(openParen, closeParen):
node = ListNode(openParen.pos)
node.value = openParen
node.next = closeParen
return node
return Parser.allOf(
Parser.terminalParser(TokenType.OPEN_PAREN),
Parser.terminalParser(TokenType.CLOSE_PAREN),
createNode=emptyList
)
# '[' expr listTail
@staticmethod
def _parser2():
def createNode(openParen, expr, listTail):
node = ListNode(openParen.pos)
node.value = expr
node.next = listTail
return node
return Parser.allOf(
Parser.terminalParser(TokenType.OPEN_PAREN, lambda v, pos: Node(pos)),
ExpressionNode.parse,
ListTailNode.parse,
createNode=createNode
)

75
smnp/newast/node/model.py Normal file
View File

@@ -0,0 +1,75 @@
class Node:
def __init__(self, pos, children=None):
if children is None:
children = []
self.children = children
self.pos = pos
self.parent = None
for child in self.children:
if isinstance(child, Node):
child.parent = self
def __len__(self):
return len(self.children)
def __getitem__(self, key):
return self.children[key]
def __setitem__(self, key, value):
if isinstance(value, Node):
value.parent = self
self.children[key] = value
def append(self, node):
if isinstance(node, Node):
node.parent = self
self.children.append(node)
def pop(self, index):
return self.children.pop(index)
@classmethod
def _parse(cls, input):
pass
@classmethod
def parse(cls, input):
result = cls._parse(input)
if result is None:
return ParseResult.FAIL()
if not isinstance(result, ParseResult):
raise RuntimeError(f"_parse() method of '{cls.__name__}' class haven't returned ParseResult object")
return result
def print(self):
self._print(first=True)
def _print(self, prefix="", last=True, first=False):
print(prefix, '' if first else '└─' if last else '├─', self.__class__.__name__, sep="")
prefix += ' ' if last else ''
for i, child in enumerate(self.children):
last = i == len(self.children) - 1
if isinstance(child, Node):
child._print(prefix, last)
else:
print(prefix, '' if last else '', f"'{str(child)}'", sep="")
class ParseResult():
def __init__(self, result, node):
if result and node is None:
raise RuntimeError("Node musn't be None if result is set to True for ParseResult")
self.result = result
self.node = node
@staticmethod
def FAIL():
return ParseResult(False, None)
@staticmethod
def OK(node):
return ParseResult(True, node)

9
smnp/newast/node/none.py Normal file
View File

@@ -0,0 +1,9 @@
from smnp.newast.node.model import Node
class NoneNode(Node):
def __init__(self):
super().__init__((-1, -1))
def _parse(self, input):
pass

View File

@@ -0,0 +1,24 @@
from smnp.error.syntax import SyntaxException
from smnp.newast.node.expression import ExpressionNode
from smnp.newast.node.model import Node, ParseResult
from smnp.newast.parser import Parser
class Program(Node):
def __init__(self):
super().__init__((-1, -1))
@classmethod
def _parse(cls, input):
def parseToken(input):
return Parser.oneOf(
ExpressionNode.parse,
exception = SyntaxException("Unknown statement")
)(input)
root = Program()
while input.hasCurrent():
result = parseToken(input)
if result.result:
root.append(result.node)
return ParseResult.OK(root)

View File

@@ -0,0 +1,11 @@
from smnp.newast.node.expression import ExpressionNode
from smnp.newast.parser import Parser
from smnp.token.type import TokenType
class StringLiteralNode(ExpressionNode):
@classmethod
def _parse(cls, input):
createNode = lambda v, pos: StringLiteralNode.withValue(pos, v[1:len(v)-1])
return Parser.terminalParser(TokenType.STRING, createNode)(input)

75
smnp/newast/parser.py Normal file
View File

@@ -0,0 +1,75 @@
from smnp.newast.node.ignore import IgnoredNode
from smnp.newast.node.model import ParseResult
class Parser:
@staticmethod
def nonTerminalParser(parser, createNode):
def parse(input):
token = input.current()
result = parse(input)
if result.result:
return ParseResult.OK(createNode(result.node, token.pos))
return ParseResult.FAIL()
return parse
@staticmethod
def terminalParser(expectedType, createNode=None):
def provideNode(value, pos):
if createNode is None:
return IgnoredNode(pos)
return createNode(value, pos)
def parse(input):
if input.hasCurrent() and input.current().type == expectedType:
token = input.current()
input.ahead()
return ParseResult.OK(provideNode(token.value, token.pos))
return ParseResult.FAIL()
return parse
@staticmethod
def oneOf(*parsers, exception=None):
def combinedParser(input):
snap = input.snapshot()
for parser in parsers:
value = parser(input)
if value.result:
return value
if exception is not None:
raise exception
input.reset(snap)
return ParseResult.FAIL()
return combinedParser
@staticmethod
def allOf(*parsers, createNode, exception=None):
if len(parsers) == 0:
raise RuntimeError("Pass one parser at least")
def extendedParser(input):
snap = input.snapshot()
results = []
for parser in parsers:
result = parser(input)
if not result.result:
if exception is not None:
raise exception
input.reset(snap)
return ParseResult.FAIL()
results.append(result.node)
return ParseResult.OK(createNode(*results))
return extendedParser

8
smnp/newast/tools.py Normal file
View File

@@ -0,0 +1,8 @@
from smnp.error.syntax import SyntaxException
def assertToken(expected, input):
if not input.hasCurrent():
raise SyntaxException(f"Expected '{expected}'")
if expected != input.current().type:
raise SyntaxException(f"Expected '{expected}', found '{input.current().value}'", input.current().pos)

View File

@@ -1,33 +1,42 @@
from smnp.ast.node.access import AccessNode
from smnp.ast.node.function import FunctionCallNode
from smnp.error.base import SmnpException
from smnp.error.runtime import RuntimeException
from smnp.runtime.evaluator import evaluate
from smnp.runtime.evaluators.list import evaluateList
def evaluateAccess(access, environment):
pass
#element = evaluate(access.element, environment)
# TODO: narazie tylko metody działają
#e = evaluateMethodCall(element, access.property, environment)
element = evaluate(access.element, environment)
if type(access.property) == FunctionCallNode:
return evaluateMethodCall(element, access.property, environment)
if type(access.property) == AccessNode:
return evaluateAccess(access.property, environment)
raise RuntimeException("Not implemented yet", access.property.pos)
# TODO only methods can be handled so far
def evaluateMethodCall(element, methodCall, environment):
try:
methodName = methodCall.identifier.identifier
arguments = evaluateList(methodCall.arguments, environment)
def evaluateMethodCall(element, functionCall, environment):
funcName = functionCall.identifier.identifier
arguments = evaluateList(functionCall.arguments, environment)
arguments.insert(0, element)
# for name, library in environment.customFunctions.items():
# if funcName == name:
# if len(library['params']) != len(arguments):
# raise RuntimeException(functionCall.pos, f"Calling '{funcName}' requires {len(library['params'])} and {len(arguments)} was passed")
# environment.scopes.append({ library['params'][i].identifier: v for i, v in enumerate(arguments) })
# returnValue = None
# for node in library['body']:
# if not isinstance(node, ReturnNode):
# evaluate(node, environment)
# else:
# returnValue = evaluateReturn(node, environment)
# environment.scopes.pop(-1)
# return returnValue
for name, definition in environment.methods[type(element)].items():
if name == funcName:
return definition(arguments, environment)
raise RuntimeException(f"Method '{funcName}' does not exist", functionCall.pos)
return environment.invokeMethod(methodName, element, arguments)
except SmnpException as e:
e.pos = methodCall.pos
raise e
# for name, library in environment.customFunctions.items():
# if funcName == name:
# if len(library['params']) != len(arguments):
# raise RuntimeException(functionCall.pos, f"Calling '{funcName}' requires {len(library['params'])} and {len(arguments)} was passed")
# environment.scopes.append({ library['params'][i].identifier: v for i, v in enumerate(arguments) })
# returnValue = None
# for node in library['body']:
# if not isinstance(node, ReturnNode):
# evaluate(node, environment)
# else:
# returnValue = evaluateReturn(node, environment)
# environment.scopes.pop(-1)
# return returnValue

View File

@@ -25,7 +25,10 @@ class TokenList:
if self.cursor >= len(self.tokens):
raise RuntimeError(f"Cursor points to not existing token! Cursor = {self.cursor}, len = {len(self.tokens)}")
return self.tokens[self.cursor]
def isCurrent(self, type):
return self.hasCurrent() and self.current().type == type
def next(self, number=1):
return self.tokens[self.cursor + number]
@@ -42,14 +45,13 @@ class TokenList:
self.cursor += 1
def snapshot(self):
self.snapshot = self.cursor
return self.cursor
def reset(self):
self.cursor = self.snapshot
return self.tokens[self.cursor]
def reset(self, snap):
self.cursor = snap
def __str__(self):
return f"[Cursor: {self.cursor}\n{', '.join([str(token) for token in self.tokens])}]"
return f"[Current({self.cursor}): {self.current() if self.hasCurrent() else 'out of tokens'}\n{', '.join([str(token) for token in self.tokens])}]"
def __repr__(self):
return self.__str__()