bartek/smnp-py
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Create almost working proof of concept with new parsers

Browse Source
This commit is contained in:
Bartłomiej Pluta
2019-07-11 19:10:13 +02:00
parent ed2c8dc6dd
commit e7674a4834
8 changed files with 128 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files

71
smnp/ast/node/atom.py
View File

@@ -1,6 +1,7 @@
from smnp.ast.node.model import Node from smnp.ast.node.model import Node
from smnp.ast.parser import Parser from smnp.ast.parser import Parsers
from smnp.token.type import TokenType from smnp.token.type import TokenType
from smnp.util.singleton import SingletonParser
class Atom(Node): class Atom(Node):
@@ -43,52 +44,60 @@ class TypeLiteral(Atom):
pass pass
def IntegerParser(input): @SingletonParser
return Parser.oneOf( def IntegerParser():
Parser.terminalParser(TokenType.INTEGER, lambda val, pos: IntegerLiteral.withValue(int(val), pos)), return Parsers.oneOf(
Parser.allOf( Parsers.terminal(TokenType.INTEGER, lambda val, pos: IntegerLiteral.withValue(int(val), pos)),
Parser.terminalParser(TokenType.MINUS), Parsers.allOf(
Parser.terminalParser(TokenType.INTEGER, lambda val, pos: IntegerLiteral.withValue(int(val), pos)), Parsers.terminal(TokenType.MINUS),
Parsers.terminal(TokenType.INTEGER, lambda val, pos: IntegerLiteral.withValue(int(val), pos)),
createNode=lambda minus, integer: IntegerLiteral.withValue(-integer.value, minus.pos), createNode=lambda minus, integer: IntegerLiteral.withValue(-integer.value, minus.pos),
name="negative integer" name="negativeInteger"
) ),
)(input) name="int"
)
def StringParser(input): @SingletonParser
return Parser.terminalParser(TokenType.STRING, createNode=StringLiteral.withValue)(input) def StringParser():
return Parsers.terminal(TokenType.STRING, createNode=StringLiteral.withValue)
def NoteParser(input): @SingletonParser
return Parser.terminalParser(TokenType.NOTE, createNode=NoteLiteral.withValue)(input) def NoteParser():
return Parsers.terminal(TokenType.NOTE, createNode=NoteLiteral.withValue)
def BoolParser(input): @SingletonParser
return Parser.terminalParser(TokenType.BOOL, createNode=BoolLiteral.withValue)(input) def BoolParser():
return Parsers.terminal(TokenType.BOOL, createNode=BoolLiteral.withValue)
def TypeParser(input): @SingletonParser
return Parser.terminalParser(TokenType.TYPE, createNode=TypeLiteral.withValue)(input) def TypeParser():
return Parsers.terminal(TokenType.TYPE, createNode=TypeLiteral.withValue)
def LiteralParser(input): @SingletonParser
return Parser.oneOf( def LiteralParser():
IntegerParser, return Parsers.oneOf(
StringParser, IntegerParser(),
NoteParser, StringParser(),
BoolParser, NoteParser(),
TypeParser, BoolParser(),
TypeParser(),
name="literal" name="literal"
)(input) )
def AtomParser(input): @SingletonParser
def AtomParser():
from smnp.ast.node.identifier import IdentifierParser from smnp.ast.node.identifier import IdentifierParser
return Parser.oneOf( return Parsers.oneOf(
LiteralParser, LiteralParser(),
IdentifierParser, IdentifierParser(),
name="atom" name="atom"
)(input) )

32
smnp/ast/node/chain.py
View File

@@ -1,21 +1,31 @@
from smnp.ast.node.atom import AtomParser
from smnp.ast.node.list import ListParser
from smnp.ast.node.map import MapParser
from smnp.ast.node.operator import BinaryOperator from smnp.ast.node.operator import BinaryOperator
from smnp.ast.node.valuable import Valuable from smnp.ast.node.valuable import Valuable
from smnp.ast.parser import Parser from smnp.ast.parser import Parsers
from smnp.token.type import TokenType from smnp.token.type import TokenType
from smnp.util.singleton import SingletonParser
class Chain(Valuable): class Chain(Valuable):
pass pass
itemParser = Parser.oneOf(
ListParser,
MapParser,
AtomParser,
)
ChainParser = Parser.leftAssociativeOperatorParser(itemParser, [TokenType.DOT], itemParser,
lambda left, op, right: Chain.withValue(BinaryOperator.withValues(left, op, right))) @SingletonParser
def ChainParser():
from smnp.ast.node.atom import AtomParser
itemParser = Parsers.oneOf(
#ListParser,
#MapParser,
AtomParser,
name="chainItem"
)
return Parsers.leftAssociativeOperatorParser(
itemParser,
[TokenType.DOT],
itemParser,
lambda left, op, right: Chain.withValue(BinaryOperator.withValues(left, op, right)),
name="chain"
)

33
smnp/ast/node/identifier.py
View File

@@ -1,11 +1,10 @@
from smnp.ast.node.atom import Atom from smnp.ast.node.atom import Atom
from smnp.ast.node.expression import MaxPrecedenceExpressionParser
from smnp.ast.node.iterable import abstractIterableParser
from smnp.ast.node.model import Node from smnp.ast.node.model import Node
from smnp.ast.node.none import NoneNode from smnp.ast.node.none import NoneNode
from smnp.ast.node.operator import BinaryOperator, Operator from smnp.ast.node.operator import BinaryOperator, Operator
from smnp.ast.parser import Parser from smnp.ast.parser import Parsers
from smnp.token.type import TokenType from smnp.token.type import TokenType
from smnp.util.singleton import SingletonParser
class Identifier(Atom): class Identifier(Atom):
@@ -49,24 +48,28 @@ class Assignment(BinaryOperator):
pass pass
def IdentifierParser(input): @SingletonParser
identifierLiteralParser = Parser.terminalParser(TokenType.IDENTIFIER, createNode=Identifier.withValue) def IdentifierParser():
identifierLiteralParser = Parsers.terminal(TokenType.IDENTIFIER, createNode=Identifier.withValue)
functionCallParser = Parser.allOf( functionCallParser = Parsers.allOf(
identifierLiteralParser, identifierLiteralParser,
abstractIterableParser(ArgumentsList, TokenType.OPEN_PAREN, TokenType.CLOSE_PAREN, MaxPrecedenceExpressionParser), #abstractIterableParser(ArgumentsList, TokenType.OPEN_PAREN, TokenType.CLOSE_PAREN, MaxPrecedenceExpressionParser),
createNode=lambda name, arguments: FunctionCall.withChildren(name, arguments) createNode=lambda name, arguments: FunctionCall.withChildren(name, arguments),
name="functionCall"
) )
assignmentParser = Parser.allOf( assignmentParser = Parsers.allOf(
identifierLiteralParser, identifierLiteralParser,
Parser.terminalParser(TokenType.ASSIGN, createNode=Operator.withValue), Parsers.terminal(TokenType.ASSIGN, createNode=Operator.withValue),
MaxPrecedenceExpressionParser, #MaxPrecedenceExpressionParser,
createNode=lambda identifier, assign, expr: Assignment.withValues(identifier, assign, expr) createNode=lambda identifier, assign, expr: Assignment.withValues(identifier, assign, expr),
name="assignment"
) )
return Parser.oneOf( return Parsers.oneOf(
assignmentParser, assignmentParser,
functionCallParser, functionCallParser,
identifierLiteralParser identifierLiteralParser,
)(input) name="idExpr"
)

37
smnp/ast/node/iterable.py
View File

@@ -1,11 +1,11 @@
from smnp.ast.node.ignore import IgnoredNode from smnp.ast.node.ignore import IgnoredNode
from smnp.ast.node.model import Node, ParseResult from smnp.ast.node.model import Node, ParseResult
from smnp.ast.node.none import NoneNode from smnp.ast.node.none import NoneNode
from smnp.ast.parser import Parser from smnp.ast.parser import Parsers
from smnp.token.type import TokenType from smnp.token.type import TokenType
def abstractIterableParser(iterableNodeType, openTokenType, closeTokenType, itemParser): def abstractIterableParser(iterableNodeType, openTokenType, closeTokenType, itemParser, name):
class AbstractIterable(Node): class AbstractIterable(Node):
def __init__(self, pos): def __init__(self, pos):
@@ -32,9 +32,10 @@ def abstractIterableParser(iterableNodeType, openTokenType, closeTokenType, item
pass pass
def abstractIterableParser(input): def abstractIterableParser(input):
return Parser.oneOf( return Parsers.oneOf(
emptyIterable, emptyIterable,
openIterable openIterable,
name=name
)(input) )(input)
def emptyIterable(input): def emptyIterable(input):
@@ -44,10 +45,11 @@ def abstractIterableParser(iterableNodeType, openTokenType, closeTokenType, item
node.next = close node.next = close
return node return node
return Parser.allOf( return Parsers.allOf(
Parser.terminalParser(openTokenType), Parsers.terminal(openTokenType),
Parser.terminalParser(closeTokenType), Parsers.terminal(closeTokenType),
createNode=createNode createNode=createNode,
name=name+"Empty"
)(input) )(input)
def openIterable(input): def openIterable(input):
@@ -57,17 +59,19 @@ def abstractIterableParser(iterableNodeType, openTokenType, closeTokenType, item
node.next = tail node.next = tail
return node return node
return Parser.allOf( return Parsers.allOf(
Parser.terminalParser(openTokenType), Parsers.terminal(openTokenType),
itemParser, itemParser,
abstractIterableTailParser, abstractIterableTailParser,
createNode=createNode createNode=createNode,
name=name+"Open"
)(input) )(input)
def abstractIterableTailParser(input): def abstractIterableTailParser(input):
return Parser.oneOf( return Parsers.oneOf(
closeIterable, closeIterable,
nextItem, nextItem,
name=name+"Tail"
)(input) )(input)
def nextItem(input): def nextItem(input):
@@ -77,15 +81,16 @@ def abstractIterableParser(iterableNodeType, openTokenType, closeTokenType, item
node.next = tail node.next = tail
return node return node
return Parser.allOf( return Parsers.allOf(
Parser.terminalParser(TokenType.COMMA, doAssert=True), Parsers.terminal(TokenType.COMMA, doAssert=True),
itemParser, itemParser,
abstractIterableTailParser, abstractIterableTailParser,
name=name+"NextItem",
createNode=createNode createNode=createNode
)(input) )(input)
def closeIterable(input): def closeIterable(input):
return Parser.terminalParser(closeTokenType)(input) return Parsers.terminal(closeTokenType)(input)
return toFlatDesiredNode(iterableNodeType, abstractIterableParser) return toFlatDesiredNode(iterableNodeType, abstractIterableParser)
@@ -105,7 +110,7 @@ def toFlatDesiredNode(iterableNodeType, parser):
return ParseResult.FAIL() return ParseResult.FAIL()
return Parser(parse, "flat", [parser]) #return Parser(parse, "flat", [parser])
def flattenList(node, output=None): def flattenList(node, output=None):

21
smnp/ast/node/map.py
View File

@@ -1,8 +1,7 @@
from smnp.ast.node.atom import LiteralParser
from smnp.ast.node.iterable import abstractIterableParser from smnp.ast.node.iterable import abstractIterableParser
from smnp.ast.node.model import Node from smnp.ast.node.model import Node
from smnp.ast.node.operator import BinaryOperator, Operator from smnp.ast.node.operator import BinaryOperator, Operator
from smnp.ast.parser import Parser from smnp.ast.parser import Parsers
from smnp.token.type import TokenType from smnp.token.type import TokenType
@@ -30,17 +29,19 @@ class Map(Node):
def MapParser(input): def MapParser(input):
from smnp.ast.node.expression import MaxPrecedenceExpressionParser from smnp.ast.node.atom import LiteralParser
#from smnp.ast.node.expression import MaxPrecedenceExpressionParser
keyParser = LiteralParser keyParser = LiteralParser
valueParser = MaxPrecedenceExpressionParser #valueParser = MaxPrecedenceExpressionParser
mapEntryParser = Parser.allOf( mapEntryParser = Parsers.allOf(
keyParser, keyParser,
Parser.terminalParser(TokenType.ARROW, createNode=Operator.withValue), Parsers.terminal(TokenType.ARROW, createNode=Operator.withValue),
valueParser, #valueParser,
createNode=MapEntry.withValues createNode=MapEntry.withValues,
name="mapEntry"
) )
mapParser = abstractIterableParser(Map, TokenType.OPEN_CURLY, TokenType.CLOSE_CURLY, mapEntryParser) return abstractIterableParser(Map, TokenType.OPEN_CURLY, TokenType.CLOSE_CURLY, mapEntryParser)
return Parser(mapParser, "map", [mapParser])(input)

23
smnp/main.py
View File

@@ -1,8 +1,7 @@
from smnp.ast.node.atom import AtomParser
from smnp.ast.node.model import Node from smnp.ast.node.model import Node
from smnp.ast.parser import TerminalParser, OneOfParser, AllOfParser, OptionalParser, LoopParser
from smnp.error.base import SmnpException from smnp.error.base import SmnpException
from smnp.token.tokenizer import tokenize from smnp.token.tokenizer import tokenize
from smnp.token.type import TokenType
def main(): def main():
@@ -16,24 +15,8 @@ def main():
super().__init__((-1, -1)) super().__init__((-1, -1))
self.children = children self.children = children
tokens = tokenize(['{*1^*1^}']) tokens = tokenize(['"fsfsvd" "Fsefs"'])
parser = LoopParser( parser = AtomParser()
TerminalParser(TokenType.OPEN_CURLY),
AllOfParser(
OneOfParser(
TerminalParser(TokenType.ASSIGN),
TerminalParser(TokenType.ASTERISK),
name="assignOrAsterisk"
),
OptionalParser(TerminalParser(TokenType.INTEGER), name="optInt"),
TerminalParser(TokenType.DASH),
name="aoaInt",
createNode=lambda a, b, c: TestNode([a, b, c])
),
TerminalParser(TokenType.CLOSE_CURLY),
createNode=lambda a, b, c: TestNode([a, b, c]),
name="block",
)
print(parser.grammar()) print(parser.grammar())
res = parser.parse(tokens) res = parser.parse(tokens)

0
smnp/util/__init__.py Normal file
View File

14
smnp/util/singleton.py Normal file
View File

@@ -0,0 +1,14 @@
from smnp.ast.parser import Parser
def SingletonParser(function):
def wrapper(*args, **kwargs):
if not hasattr(function, 'instance'):
function.instance = function(*args, **kwargs)
if not isinstance(function.instance, Parser):
raise RuntimeError(f"Function {function.__name__} haven't returned Parser object")
return function.instance
return wrapper