diff --git a/smnp/ast/parser.py b/smnp/ast/parser.py index 48a6435..f25a283 100644 --- a/smnp/ast/parser.py +++ b/smnp/ast/parser.py @@ -1,5 +1,6 @@ from smnp.ast.node.ignore import IgnoredNode from smnp.ast.node.model import ParseResult, Node +from smnp.ast.node.none import NoneNode from smnp.error.syntax import SyntaxException @@ -29,7 +30,12 @@ class Parser(object): def grammar(self): rules = [] self._grammarRules(rules) - return "\n".join(rules) + return "\n".join(self._uniq(rules)) + + def _uniq(self, seq): + seen = set() + seen_add = seen.add + return [x for x in seq if not (x in seen or seen_add(x))] def _grammarRules(self, output): output.append(f"class '{self.__class__.__name__}' does not implement _grammarRules() method") @@ -44,6 +50,37 @@ class Parser(object): return self.__str__() +class Parsers: + + @staticmethod + def terminal(expectedType, createNode=lambda val, pos: IgnoredNode(pos), doAssert=False): + return TerminalParser(expectedType, createNode, doAssert) + + @staticmethod + def oneOf(*parsers, name, exception=None): + return OneOfParser(*parsers, name=name, exception=exception) + + @staticmethod + def allOf(*parsers, createNode, exception=None, name): + return AllOfParser(*parsers, createNode=createNode, exception=exception, name=name) + + @staticmethod + def leftAssociativeOperatorParser(leftParser, operatorTokenTypes, rightParser, createNode, name): + return LeftAssociativeOperatorParser(leftParser, operatorTokenTypes, rightParser, createNode, name) + + @staticmethod + def many(itemParser, createNode, name): + return ManyParser(itemParser, createNode, name) + + @staticmethod + def optional(parser, name): + return OptionalParser(parser, name) + + @staticmethod + def loop(startParser, itemParser, endParser, createNode, name): + return LoopParser(startParser, itemParser, endParser, createNode, name) + + class TerminalParser(Parser): def __init__(self, expectedType, createNode=lambda val, pos: IgnoredNode(pos), doAssert=False): @@ -67,6 +104,7 @@ class TerminalParser(Parser): return ParseResult.FAIL() + class OneOfParser(Parser): def __init__(self, *parsers, name, exception=None): super().__init__(name) @@ -134,175 +172,107 @@ class AllOfParser(Parser): output.append(self.name + ' -> ' + ' '.join([ parser.name for parser in self.parsers ])) [ parser._grammarRules(output) for parser in self.parsers ] -# def allOf(*parsers, createNode, exception=None, name="all"): -# if len(parsers) == 0: -# raise RuntimeError("Pass one parser at least") -# -# def extendedParser(input): -# snap = input.snapshot() -# -# results = [] -# -# for parser in parsers: -# result = parser(input) -# -# if not result.result: -# if exception is not None: -# if callable(exception): -# raise exception(input) -# else: -# raise exception -# -# input.reset(snap) -# return ParseResult.FAIL() -# -# results.append(result.node) -# -# node = createNode(*results) -# if not isinstance(node, Node): -# raise RuntimeError("Function 'createNode' haven't returned a Node object. Probably forget to pass 'return'") -# -# return ParseResult.OK(node) -# -# -# return Parser(extendedParser, name=name, parsers=parsers) -# + +class LeftAssociativeOperatorParser(Parser): + def __init__(self, leftParser, operatorTokenTypes, rightParser, createNode, name): + from smnp.ast.node.operator import Operator + + super().__init__(name) + self.leftParser = leftParser + self.rightParser = rightParser + self.createNode = createNode + self.operators = operatorTokenTypes + + operatorParsers = [ TerminalParser(expectedType, createNode=lambda val, pos: Operator.withValue(val, pos)) for expectedType in operatorTokenTypes ] + self.operatorParser = OneOfParser(*operatorParsers, name="not important") + + def _parse(self, input): + snap = input.snapshot() + left = self.leftParser.parse(input) + if left.result: + operator = self.operatorParser.parse(input) + while operator.result: + right = self.rightParser.parse(input) + left = ParseResult.OK(self.createNode(left.node, operator.node, right.node)) + operator = self.operatorParser.parse(input) + return left + + return ParseResult.FAIL() -# def o -# # a -> A -# @staticmethod -# def terminalParser(expectedType, createNode=None, doAssert=False): -# def provideNode(value, pos): -# if createNode is None: -# return IgnoredNode(pos) -# return createNode(value, pos) -# -# def parse(input): -# if input.hasCurrent() and input.current().type == expectedType: -# token = input.current() -# input.ahead() -# return ParseResult.OK(provideNode(token.value, token.pos)) -# elif doAssert: -# found = f", found '{input.current().rawValue}'" if input.hasCurrent() else "" -# raise SyntaxException(f"Expected '{expectedType.key}'{found}", input.currentPos()) -# -# return ParseResult.FAIL() -# -# return Parser(parse, name=expectedType.name.lower()) -# -# # oneOf -> a | b | c | ... -# @staticmethod -# def oneOf(*parsers, exception=None, name="or"): -# def combinedParser(input): -# snap = input.snapshot() -# for parser in parsers: -# value = parser(input) -# if value.result: -# return value -# -# if exception is not None: -# if callable(exception): -# raise exception(input) -# else: -# raise exception -# -# -# input.reset(snap) -# return ParseResult.FAIL() -# -# return Parser(combinedParser, name=name, parsers=parsers) -# -# # allOf -> a b c ... -# @staticmethod + def _grammarRules(self, output): + output.append('\n'.join([f"{self.name} -> {self.leftParser.name} {operator.name.lower()} {self.rightParser.name} | {self.leftParser.name}" for operator in self.operators])) + self.leftParser._grammarRules(output) + self.rightParser._grammarRules(output) + + +class ManyParser(Parser): + def __init__(self, itemParser, createNode, name): + super().__init__(name) + self.itemParser = itemParser + self.createNode = createNode + + def _parse(self, input): + snap = input.snapshot() + + parsedItems = [] + pos = input.currentPos() + while True: + result = self.itemParser.parse(input) + if result.result: + parsedItems.append(result.node) + snap = input.snapshot() + else: + input.reset(snap) + return ParseResult.OK(self.createNode(parsedItems, pos) if len(parsedItems) > 0 else NoneNode()) + + def _grammarRules(self, output): + output.append(f"{self.name} -> {self.itemParser.name}*") + self.itemParser._grammarRules(output) + + +class OptionalParser(Parser): + def __init__(self, parser, name): + super().__init__(name) + self.parser = parser + + def _parse(self, input): + result = self.parser.parse(input) + if result.result: + return result + + return ParseResult.OK(NoneNode()) + + def _grammarRules(self, output): + output.append(f"{self.name} -> {self.parser.name}?") + + +class LoopParser(Parser): + def __init__(self, startParser, itemParser, endParser, createNode, name): + super().__init__(name) + self.startParser = startParser + self.itemParser = itemParser + self.endParser = endParser + self.createNode = createNode + + def _parse(self, input): + items = [] + start = self.startParser.parse(input) + if start.result: + while True: + end = self.endParser.parse(input) + if end.result: + return ParseResult.OK(self.createNode(start.node, items, end.node)) + item = self.itemParser.parse(input) + if not item.result: + return ParseResult.FAIL() + items.append(item.node) + + return ParseResult.FAIL() + + def _grammarRules(self, output): + output.append(f"{self.name} -> {self.startParser.name} {self.itemParser.name}* {self.endParser.name}") + self.startParser._grammarRules(output) + self.itemParser._grammarRules(output) + self.endParser._grammarRules(output) -# -# # leftAssociative -> left | left OP right -# @staticmethod -# def leftAssociativeOperatorParser(leftParser, operatorTokenTypes, rightParser, createNode, name="leftAssoc"): -# from smnp.ast.node.operator import Operator -# -# def parse(input): -# operatorParser = Parser.oneOfTerminals(*operatorTokenTypes, createNode=lambda val, pos: Operator.withChildren([val], pos)) -# left = leftParser(input) -# if left.result: -# operator = operatorParser(input) -# while operator.result: -# right = rightParser(input) -# left = ParseResult.OK(createNode(left.node, operator.node, right.node)) -# operator = operatorParser(input) -# return left -# -# return ParseResult.FAIL() -# -# return Parser(parse, name=name, parsers=[leftParser, '|'.join([t.value for t in operatorTokenTypes]), rightParser]) -# -# @staticmethod -# def oneOfTerminals(*tokenTypes, createNode=None): -# return Parser.oneOf(*[ Parser.terminalParser(expectedType, createNode=createNode) for expectedType in tokenTypes ], name='|'.join([t.value for t in tokenTypes])) -# -# # loop -> start item* end -# @staticmethod -# def loop(startParser, itemParser, endParser, createNode, name="loop"): -# def parse(input): -# items = [] -# start = startParser(input) -# if start.result: -# while True: -# end = endParser(input) -# if end.result: -# return ParseResult.OK(createNode(start.node, items, end.node)) -# item = itemParser(input) -# if not item.result: -# return ParseResult.FAIL() -# items.append(item.node) -# -# return ParseResult.FAIL() -# -# return Parser(parse, name, parsers=[startParser, itemParser, endParser]) -# -# @staticmethod -# def doAssert(parser, expected, name="!!"): -# def parse(input): -# result = parser(input) -# -# if not result.result: -# found = f", found '{input.current().rawValue}'" if input.hasCurrent() else '' -# -# raise SyntaxException(f"Expected {expected}{found}", input.currentPos()) -# -# return result -# -# return Parser(parse, name, parsers=parser) -# -# @staticmethod -# def optional(parser, name="??"): -# def parse(input): -# result = parser(input) -# if result.result: -# return result -# -# return ParseResult.OK(NoneNode()) -# -# return Parser(parse, name, parsers=[parser]) -# -# @staticmethod -# def epsilon(): -# return lambda *args: ParseResult.OK(NoneNode()) -# -# @staticmethod -# def many(parser, createNode, name="*"): -# def parse(input): -# results = [] -# snap = input.snapshot() -# pos = input.currentPos() -# while True: -# result = parser(input) -# if result.result: -# results.append(result.node) -# snap = input.snapshot() -# else: -# input.reset(snap) -# return ParseResult.OK(createNode(results, pos) if len(results) > 0 else NoneNode()) -# -# return Parser(parse, name, parsers=[parser]) diff --git a/smnp/main.py b/smnp/main.py index 10109f4..0763671 100644 --- a/smnp/main.py +++ b/smnp/main.py @@ -1,5 +1,5 @@ from smnp.ast.node.model import Node -from smnp.ast.parser import TerminalParser, OneOfParser, AllOfParser +from smnp.ast.parser import TerminalParser, OneOfParser, AllOfParser, OptionalParser, LoopParser from smnp.error.base import SmnpException from smnp.token.tokenizer import tokenize from smnp.token.type import TokenType @@ -7,30 +7,36 @@ from smnp.token.type import TokenType def main(): try: - #stdLibraryEnv = loadStandardLibrary() - #Interpreter.interpretFile(sys.argv[1], printTokens=True, printAst=True, execute=False, baseEnvironment=None) - #draft() + # stdLibraryEnv = loadStandardLibrary() + # Interpreter.interpretFile(sys.argv[1], printTokens=True, printAst=True, execute=False, baseEnvironment=None) + # draft() - def node(*items): - n = Node((-1, -1)) - n.children = items - return n + class TestNode(Node): + def __init__(self, children): + super().__init__((-1, -1)) + self.children = children - tokens = tokenize(['=x']) - parser = AllOfParser( - OneOfParser( - TerminalParser(TokenType.ASSIGN), - TerminalParser(TokenType.ASTERISK), - name="assignOrAsterisk" + tokens = tokenize(['{*1^*1^}']) + parser = LoopParser( + TerminalParser(TokenType.OPEN_CURLY), + AllOfParser( + OneOfParser( + TerminalParser(TokenType.ASSIGN), + TerminalParser(TokenType.ASTERISK), + name="assignOrAsterisk" + ), + OptionalParser(TerminalParser(TokenType.INTEGER), name="optInt"), + TerminalParser(TokenType.DASH), + name="aoaInt", + createNode=lambda a, b, c: TestNode([a, b, c]) ), - TerminalParser(TokenType.INTEGER), - name="aoaInt", - createNode=node + TerminalParser(TokenType.CLOSE_CURLY), + createNode=lambda a, b, c: TestNode([a, b, c]), + name="block", ) print(parser.grammar()) res = parser.parse(tokens) - print() if res.result: res.node.print()