[BIG REFACTOR] Create new project structure and prepare scaffolding for external modules system

This commit is contained in:
2020-03-09 19:30:43 +01:00
parent ae9406cc50
commit dcdecee28f
155 changed files with 290 additions and 192 deletions

View File

@@ -0,0 +1,5 @@
package io.smnp
fun main(args: Array<String>) {
}

View File

@@ -0,0 +1,20 @@
package io.smnp.dsl.ast.model.entity
import io.smnp.dsl.ast.model.enumeration.ParsingResult
import io.smnp.dsl.ast.model.node.Node
class ParserOutput private constructor(val result: ParsingResult, val node: Node) {
fun map(mapper: (Node) -> Node): ParserOutput {
return if(result == ParsingResult.OK) ok(mapper(node)) else fail()
}
companion object {
fun ok(node: Node): ParserOutput {
return ParserOutput(ParsingResult.OK, node)
}
fun fail(): ParserOutput {
return ParserOutput(ParsingResult.FAILED, Node.NONE)
}
}
}

View File

@@ -0,0 +1,6 @@
package io.smnp.dsl.ast.model.enumeration
enum class ParsingResult {
OK,
FAILED
}

View File

@@ -0,0 +1,8 @@
package io.smnp.dsl.ast.model.node
import io.smnp.dsl.token.model.entity.TokenPosition
abstract class AbstractIterableNode(items: List<Node>, position: TokenPosition) : Node(items, position) {
val items: List<Node>
get() = children
}

View File

@@ -0,0 +1,3 @@
package io.smnp.dsl.ast.model.node
class AccessOperatorNode(lhs: Node, operator: Node, rhs: Node) : BinaryOperatorAbstractNode(lhs, operator, rhs)

View File

@@ -0,0 +1,3 @@
package io.smnp.dsl.ast.model.node
class AssignmentOperatorNode(lhs: Node, operator: Node, rhs: Node) : BinaryOperatorAbstractNode(lhs, operator, rhs)

View File

@@ -0,0 +1,11 @@
package io.smnp.dsl.ast.model.node
import io.smnp.dsl.token.model.entity.Token
abstract class AtomAbstractNode(val token: Token) : Node(1, token.position) {
override fun pretty(prefix: String, last: Boolean, first: Boolean) {
println(prefix + (if (first) "" else if (last) "└─" else "├─") + this::class.simpleName + " " + position)
println(prefix + (if (last) " " else "") + "" + token)
}
}

View File

@@ -0,0 +1,22 @@
package io.smnp.dsl.ast.model.node
abstract class BinaryOperatorAbstractNode(lhs: Node, operator: Node, rhs: Node) : Node(3, operator.position) {
operator fun component1() = children[0]
operator fun component2() = children[1]
operator fun component3() = children[2]
val lhs: Node
get() = children[0]
val operator: Node
get() = children[1]
val rhs: Node
get() = children[2]
init {
children[0] = lhs
children[1] = operator
children[2] = rhs
}
}

View File

@@ -0,0 +1,10 @@
package io.smnp.dsl.ast.model.node
class BlockNode(begin: Node, statements: List<Node>, end: Node) : Node(statements, begin.position) {
val statements: List<Node>
get() = children
init {
children = statements.toMutableList()
}
}

View File

@@ -0,0 +1,5 @@
package io.smnp.dsl.ast.model.node
import io.smnp.dsl.token.model.entity.Token
class BoolLiteralNode(token: Token) : AtomAbstractNode(token)

View File

@@ -0,0 +1,22 @@
package io.smnp.dsl.ast.model.node
class ConditionNode(trueBranchToken: Node, condition: Node, trueBranch: Node, falseBranchToken: Node, falseBranch: Node) : Node(3, trueBranchToken.position) {
operator fun component1(): Node = children[0]
operator fun component2(): Node = children[1]
operator fun component3(): Node = children[2]
val condition: Node
get() = children[0]
val trueBranch: Node
get() = children[1]
val falseBranch: Node
get() = children[2]
init {
children[0] = condition
children[1] = trueBranch
children[2] = falseBranch
}
}

View File

@@ -0,0 +1,20 @@
package io.smnp.dsl.ast.model.node
import io.smnp.dsl.token.model.entity.TokenPosition
class ExtendNode(type: Node, identifier: Node, functions: Node, position: TokenPosition) : Node(3, position) {
val type: Node
get() = children[0]
val identifier: Node
get() = children[1]
val functions: Node
get() = children[2]
init {
children[0] = type
children[1] = identifier
children[2] = functions
}
}

View File

@@ -0,0 +1,5 @@
package io.smnp.dsl.ast.model.node
import io.smnp.dsl.token.model.entity.Token
class FloatLiteralNode(token: Token) : AtomAbstractNode(token)

View File

@@ -0,0 +1,5 @@
package io.smnp.dsl.ast.model.node
import io.smnp.dsl.token.model.entity.TokenPosition
class FunctionCallArgumentsNode(items: List<Node>, position: TokenPosition) : AbstractIterableNode(items, position)

View File

@@ -0,0 +1,14 @@
package io.smnp.dsl.ast.model.node
class FunctionCallNode(identifier: Node, arguments: Node) : Node(2, identifier.position) {
val identifier: Node
get() = children[0]
val arguments: Node
get() = children[1]
init {
children[0] = identifier
children[1] = arguments
}
}

View File

@@ -0,0 +1,5 @@
package io.smnp.dsl.ast.model.node
import io.smnp.dsl.token.model.entity.TokenPosition
class FunctionDefinitionArgumentsNode(items: List<Node>, position: TokenPosition) : AbstractIterableNode(items, position)

View File

@@ -0,0 +1,20 @@
package io.smnp.dsl.ast.model.node
import io.smnp.dsl.token.model.entity.TokenPosition
class FunctionDefinitionNode(identifier: Node, arguments: Node, body: Node, position: TokenPosition) : Node(3, position) {
val identifier: Node
get() = children[0]
val arguments: Node
get() = children[1]
val body: Node
get() = children[2]
init {
children[0] = identifier
children[1] = arguments
children[2] = body
}
}

View File

@@ -0,0 +1,5 @@
package io.smnp.dsl.ast.model.node
import io.smnp.dsl.token.model.entity.Token
class IdentifierNode(token: Token) : AtomAbstractNode(token)

View File

@@ -0,0 +1,12 @@
package io.smnp.dsl.ast.model.node
import io.smnp.dsl.token.model.entity.TokenPosition
class ImportNode(path: Node, position: TokenPosition) : Node(1, position) {
val path: Node
get() = children[0]
init {
children[0] = path
}
}

View File

@@ -0,0 +1,5 @@
package io.smnp.dsl.ast.model.node
import io.smnp.dsl.token.model.entity.Token
class IntegerLiteralNode(token: Token) : AtomAbstractNode(token)

View File

@@ -0,0 +1,5 @@
package io.smnp.dsl.ast.model.node
import io.smnp.dsl.token.model.entity.TokenPosition
class ListNode(items: List<Node>, position: TokenPosition) : AbstractIterableNode(items, position)

View File

@@ -0,0 +1,3 @@
package io.smnp.dsl.ast.model.node
class LogicOperatorNode(lhs: Node, operator: Node, rhs: Node) : BinaryOperatorAbstractNode(lhs, operator, rhs)

View File

@@ -0,0 +1,22 @@
package io.smnp.dsl.ast.model.node
class LoopNode(iterator: Node, parameters: Node, operator: Node, statement: Node, filter: Node): Node(4, operator.position) {
val iterator: Node
get() = children[0]
val parameters: Node
get() = children[1]
val statement: Node
get() = children[2]
val filter: Node
get() = children[3]
init {
children[0] = iterator
children[1] = parameters
children[2] = statement
children[3] = filter
}
}

View File

@@ -0,0 +1,5 @@
package io.smnp.dsl.ast.model.node
import io.smnp.dsl.token.model.entity.TokenPosition
class LoopParametersNode(items: List<Node>, position: TokenPosition) : AbstractIterableNode(items, position)

View File

@@ -0,0 +1,14 @@
package io.smnp.dsl.ast.model.node
class MapEntryNode(key: Node, operator: Node, value: Node) : Node(2, operator.position) {
val key: Node
get() = children[0]
val value: Node
get() = children[1]
init {
children[0] = key
children[1] = value
}
}

View File

@@ -0,0 +1,5 @@
package io.smnp.dsl.ast.model.node
import io.smnp.dsl.token.model.entity.TokenPosition
class MapNode(items: List<Node>, position: TokenPosition) : AbstractIterableNode(items, position)

View File

@@ -0,0 +1,3 @@
package io.smnp.dsl.ast.model.node
class MinusOperatorNode(operator: Node, operand: Node) : UnaryOperatorAbstractNode(operator, operand)

View File

@@ -0,0 +1,35 @@
package io.smnp.dsl.ast.model.node
import io.smnp.dsl.token.model.entity.TokenPosition
abstract class Node(numberOfChildren: Int, val position: TokenPosition) {
var children: MutableList<Node> = MutableList(numberOfChildren) { NONE }
protected set
constructor(children: List<Node>, position: TokenPosition) : this(children.size, position) {
this.children = children.toMutableList()
}
operator fun get(index: Int) = children[index]
operator fun set(index: Int, value: Node) {
children[index] = value
}
open fun pretty(prefix: String = "", last: Boolean = true, first: Boolean = true) {
var newPrefix = prefix
var newLast = last
val nodeName = this::class.simpleName ?: "<anonymous>"
println(newPrefix + (if (first) "" else if (newLast) "└─" else "├─") + nodeName + " " + position)
newPrefix += if (newLast) " " else ""
for ((index, child) in children.withIndex()) {
newLast = index == children.size - 1
child.pretty(newPrefix, newLast, false)
}
}
companion object {
val NONE = NoneNode()
}
}

View File

@@ -0,0 +1,7 @@
package io.smnp.dsl.ast.model.node
import io.smnp.dsl.token.model.entity.TokenPosition
class NoneNode : Node(0,
TokenPosition.NONE
)

View File

@@ -0,0 +1,3 @@
package io.smnp.dsl.ast.model.node
class NotOperatorNode(operator: Node, operand: Node) : UnaryOperatorAbstractNode(operator, operand)

View File

@@ -0,0 +1,5 @@
package io.smnp.dsl.ast.model.node
import io.smnp.dsl.token.model.entity.Token
class NoteLiteralNode(token: Token) : AtomAbstractNode(token)

View File

@@ -0,0 +1,18 @@
package io.smnp.dsl.ast.model.node
class OptionalFunctionDefinitionArgumentNode(identifier: Node, type: Node, defaultValue: Node) : Node(3, identifier.position) {
val identifier: Node
get() = children[0]
val type: Node
get() = children[1]
val defaultValue: Node
get() = children[2]
init {
children[0] = identifier
children[1] = type
children[2] = defaultValue
}
}

View File

@@ -0,0 +1,3 @@
package io.smnp.dsl.ast.model.node
class PowerOperatorNode(lhs: Node, operator: Node, rhs: Node) : BinaryOperatorAbstractNode(lhs, operator, rhs)

View File

@@ -0,0 +1,3 @@
package io.smnp.dsl.ast.model.node
class ProductOperatorNode(lhs: Node, operator: Node, rhs: Node) : BinaryOperatorAbstractNode(lhs, operator, rhs)

View File

@@ -0,0 +1,18 @@
package io.smnp.dsl.ast.model.node
class RegularFunctionDefinitionArgumentNode(identifier: Node, type: Node, vararg: Node) : Node(3, identifier.position) {
val identifier
get() = children[0]
val type
get() = children[1]
val vararg
get() = children[2]
init {
children[0] = identifier
children[1] = type
children[2] = vararg
}
}

View File

@@ -0,0 +1,3 @@
package io.smnp.dsl.ast.model.node
class RelationOperatorNode(lhs: Node, operator: Node, rhs: Node) : BinaryOperatorAbstractNode(lhs, operator, rhs)

View File

@@ -0,0 +1,10 @@
package io.smnp.dsl.ast.model.node
class ReturnNode(value: Node) : Node(1, value.position) {
val value: Node
get() = children[0]
init {
children[0] = value
}
}

View File

@@ -0,0 +1,5 @@
package io.smnp.dsl.ast.model.node
import io.smnp.dsl.token.model.entity.TokenPosition
class RootNode(nodes: List<Node>, position: TokenPosition) : Node(nodes, position)

View File

@@ -0,0 +1,5 @@
package io.smnp.dsl.ast.model.node
import io.smnp.dsl.token.model.entity.Token
class StringLiteralNode(token: Token) : AtomAbstractNode(token)

View File

@@ -0,0 +1,3 @@
package io.smnp.dsl.ast.model.node
class SumOperatorNode(lhs: Node, operator: Node, rhs: Node) : BinaryOperatorAbstractNode(lhs, operator, rhs)

View File

@@ -0,0 +1,10 @@
package io.smnp.dsl.ast.model.node
class ThrowNode(value: Node) : Node(1, value.position) {
val value: Node
get() = children[0]
init {
children[0] = value
}
}

View File

@@ -0,0 +1,5 @@
package io.smnp.dsl.ast.model.node
import io.smnp.dsl.token.model.entity.Token
class TokenNode(token: Token) : AtomAbstractNode(token)

View File

@@ -0,0 +1,14 @@
package io.smnp.dsl.ast.model.node
class TypeNode(type: Node, specifiers: Node) : Node(2, type.position) {
val type: Node
get() = children[0]
val specifiers: Node
get() = children[1]
init {
children[0] = type
children[1] = specifiers
}
}

View File

@@ -0,0 +1,5 @@
package io.smnp.dsl.ast.model.node
import io.smnp.dsl.token.model.entity.TokenPosition
class TypeSpecifierNode(items: List<Node>, position: TokenPosition) : AbstractIterableNode(items, position)

View File

@@ -0,0 +1,5 @@
package io.smnp.dsl.ast.model.node
import io.smnp.dsl.token.model.entity.TokenPosition
class TypeSpecifiersNode(specifiers: List<Node>, position: TokenPosition) : Node(specifiers, position)

View File

@@ -0,0 +1,17 @@
package io.smnp.dsl.ast.model.node
abstract class UnaryOperatorAbstractNode(operator: Node, operand: Node) : Node(2, operator.position) {
operator fun component1() = children[0]
operator fun component2() = children[1]
val operator: Node
get() = children[0]
val operand: Node
get() = children[1]
init {
children[0] = operator
children[1] = operand
}
}

View File

@@ -0,0 +1,41 @@
package io.smnp.dsl.ast.parser
import io.smnp.dsl.ast.model.entity.ParserOutput
import io.smnp.dsl.ast.model.node.Node
import io.smnp.dsl.token.model.entity.TokenList
import io.smnp.dsl.token.model.entity.TokenPosition
import io.smnp.dsl.token.model.enumeration.TokenType
abstract class AbstractIterableParser(
private val beginTokenType: TokenType,
private val itemParser: Parser,
private val endTokenType: TokenType,
private val createNode: (List<Node>, TokenPosition) -> Node
) : Parser() {
override fun tryToParse(input: TokenList): ParserOutput {
val emptyIterableParser = allOf(
terminal(beginTokenType),
terminal(endTokenType)
) { createNode(listOf(), it[0].position) }
val nonEmptyIterableParser = allOf(
terminal(beginTokenType),
allOf(
itemParser,
optional(repeat(allOf(
terminal(TokenType.COMMA),
itemParser
) { it[1] }) { list, position -> object : Node(list, position) {} }
)) { list ->
object : Node(listOf(list[0]) + list[1].children, TokenPosition.NONE) {}
},
terminal(endTokenType)
) { createNode(it[1].children.toList(), it[0].position) }
return oneOf(
emptyIterableParser,
nonEmptyIterableParser
).parse(input)
}
}

View File

@@ -0,0 +1,18 @@
package io.smnp.dsl.ast.parser
import io.smnp.dsl.ast.model.entity.ParserOutput
import io.smnp.dsl.ast.model.node.AssignmentOperatorNode
import io.smnp.dsl.token.model.entity.TokenList
import io.smnp.dsl.token.model.enumeration.TokenType
class AssignmentOperatorParser : Parser() {
override fun tryToParse(input: TokenList): ParserOutput {
return allOf(
SimpleIdentifierParser(),
terminal(TokenType.ASSIGN),
assert(ExpressionParser(), "expression")
) {
AssignmentOperatorNode(it[0], it[1], it[2])
}.parse(input)
}
}

View File

@@ -0,0 +1,27 @@
package io.smnp.dsl.ast.parser
import io.smnp.dsl.ast.model.entity.ParserOutput
import io.smnp.dsl.token.model.entity.TokenList
import io.smnp.dsl.token.model.enumeration.TokenType
class AtomParser : Parser() {
override fun tryToParse(input: TokenList): ParserOutput {
val parenthesesParser = allOf(
terminal(TokenType.OPEN_PAREN),
ExpressionParser(),
terminal(TokenType.CLOSE_PAREN)
) {
it[1]
}
val literalParser = oneOf(
parenthesesParser,
ComplexIdentifierParser(),
ListParser(),
LiteralParser(),
MapParser()
)
return literalParser.parse(input)
}
}

View File

@@ -0,0 +1,18 @@
package io.smnp.dsl.ast.parser
import io.smnp.dsl.ast.model.entity.ParserOutput
import io.smnp.dsl.ast.model.node.BlockNode
import io.smnp.dsl.token.model.entity.TokenList
import io.smnp.dsl.token.model.enumeration.TokenType
class BlockParser : Parser() {
override fun tryToParse(input: TokenList): ParserOutput {
return loop(
terminal(TokenType.OPEN_CURLY),
assert(StatementParser(), "statement or }"),
terminal(TokenType.CLOSE_CURLY)
) { begin, list, end ->
BlockNode(begin, list, end)
}.parse(input)
}
}

View File

@@ -0,0 +1,12 @@
package io.smnp.dsl.ast.parser
import io.smnp.dsl.ast.model.entity.ParserOutput
import io.smnp.dsl.ast.model.node.BoolLiteralNode
import io.smnp.dsl.token.model.entity.TokenList
import io.smnp.dsl.token.model.enumeration.TokenType
class BoolLiteralParser : Parser() {
override fun tryToParse(input: TokenList): ParserOutput {
return terminal(TokenType.BOOL) { BoolLiteralNode(it) }.parse(input)
}
}

View File

@@ -0,0 +1,14 @@
package io.smnp.dsl.ast.parser
import io.smnp.dsl.ast.model.entity.ParserOutput
import io.smnp.dsl.token.model.entity.TokenList
class ComplexIdentifierParser : Parser() {
override fun tryToParse(input: TokenList): ParserOutput {
return oneOf(
AssignmentOperatorParser(),
FunctionCallParser(),
SimpleIdentifierParser()
).parse(input)
}
}

View File

@@ -0,0 +1,38 @@
package io.smnp.dsl.ast.parser
import io.smnp.dsl.ast.model.entity.ParserOutput
import io.smnp.dsl.ast.model.node.ConditionNode
import io.smnp.dsl.ast.model.node.Node
import io.smnp.dsl.token.model.entity.TokenList
import io.smnp.dsl.token.model.enumeration.TokenType
class ConditionParser : Parser() {
override fun tryToParse(input: TokenList): ParserOutput {
val ifStatementParser = allOf(
terminal(TokenType.IF),
terminal(TokenType.OPEN_PAREN),
SubexpressionParser(),
terminal(TokenType.CLOSE_PAREN),
StatementParser()
) {
ConditionNode(it[0], it[2], it[4], Node.NONE, Node.NONE)
}
val ifElseStatementParser = allOf(
terminal(TokenType.IF),
terminal(TokenType.OPEN_PAREN),
SubexpressionParser(),
terminal(TokenType.CLOSE_PAREN),
StatementParser(),
terminal(TokenType.ELSE),
StatementParser()
) {
ConditionNode(it[0], it[2], it[4], it[5], it[6])
}
return oneOf(
ifElseStatementParser,
ifStatementParser
).parse(input)
}
}

View File

@@ -0,0 +1,13 @@
package io.smnp.dsl.ast.parser
import io.smnp.dsl.ast.model.entity.ParserOutput
import io.smnp.dsl.token.model.entity.TokenList
class ExpressionParser : Parser() {
override fun tryToParse(input: TokenList): ParserOutput {
return oneOf(
LoopParser(),
SubexpressionParser()
).parse(input)
}
}

View File

@@ -0,0 +1,40 @@
package io.smnp.dsl.ast.parser
import io.smnp.dsl.ast.model.entity.ParserOutput
import io.smnp.dsl.ast.model.node.BlockNode
import io.smnp.dsl.ast.model.node.ExtendNode
import io.smnp.dsl.ast.model.node.Node
import io.smnp.dsl.token.model.entity.TokenList
import io.smnp.dsl.token.model.enumeration.TokenType
class ExtendParser : Parser() {
override fun tryToParse(input: TokenList): ParserOutput {
val simpleExtendParser = allOf(
terminal(TokenType.EXTEND),
assert(TypeParser(), "type to be extended"),
assert(terminal(TokenType.AS), "'as' keyword with identifier"),
assert(SimpleIdentifierParser(), "identifier"),
terminal(TokenType.WITH),
assert(mapNode(FunctionDefinitionParser()) { BlockNode(Node.NONE, listOf(it), Node.NONE) }, "method definition")
) {
ExtendNode(it[1], it[3], it[5], it[0].position)
}
val complexExtendParser = allOf(
terminal(TokenType.EXTEND),
assert(TypeParser(), "type to be extended"),
assert(terminal(TokenType.AS), "'as' keyword with identifier"),
assert(SimpleIdentifierParser(), "identifier"),
assert(loop(terminal(TokenType.OPEN_CURLY), assert(FunctionDefinitionParser(), "method definition or }"), terminal(TokenType.CLOSE_CURLY)) {
begin, methods, end -> BlockNode(begin, methods, end)
}, "block with methods' definitions or 'with' keyword with single method definition")
) {
ExtendNode(it[1], it[3], it[4], it[0].position)
}
return oneOf(
simpleExtendParser,
complexExtendParser
).parse(input)
}
}

View File

@@ -0,0 +1,31 @@
package io.smnp.dsl.ast.parser
import io.smnp.dsl.ast.model.entity.ParserOutput
import io.smnp.dsl.ast.model.node.NotOperatorNode
import io.smnp.dsl.ast.model.node.PowerOperatorNode
import io.smnp.dsl.token.model.entity.TokenList
import io.smnp.dsl.token.model.enumeration.TokenType
class FactorParser : Parser() {
override fun tryToParse(input: TokenList): ParserOutput {
val factorParser = leftAssociativeOperator(
UnitParser(),
listOf(TokenType.DOUBLE_ASTERISK),
UnitParser()
) { lhs, operator, rhs ->
PowerOperatorNode(lhs, operator, rhs)
}
val notOperatorParser = allOf(
terminal(TokenType.NOT),
factorParser
) {
NotOperatorNode(it[0], it[1])
}
return oneOf(
notOperatorParser,
factorParser
).parse(input)
}
}

View File

@@ -0,0 +1,12 @@
package io.smnp.dsl.ast.parser
import io.smnp.dsl.ast.model.entity.ParserOutput
import io.smnp.dsl.ast.model.node.FloatLiteralNode
import io.smnp.dsl.token.model.entity.TokenList
import io.smnp.dsl.token.model.enumeration.TokenType
class FloatLiteralParser : Parser() {
override fun tryToParse(input: TokenList): ParserOutput {
return terminal(TokenType.FLOAT) { FloatLiteralNode(it) }.parse(input)
}
}

View File

@@ -0,0 +1,9 @@
package io.smnp.dsl.ast.parser
import io.smnp.dsl.ast.model.node.FunctionCallArgumentsNode
import io.smnp.dsl.token.model.enumeration.TokenType
class FunctionCallArgumentsParser :
AbstractIterableParser(TokenType.OPEN_PAREN, ExpressionParser(), TokenType.CLOSE_PAREN, { list, tokenPosition ->
FunctionCallArgumentsNode(list, tokenPosition)
})

View File

@@ -0,0 +1,16 @@
package io.smnp.dsl.ast.parser
import io.smnp.dsl.ast.model.entity.ParserOutput
import io.smnp.dsl.ast.model.node.FunctionCallNode
import io.smnp.dsl.token.model.entity.TokenList
class FunctionCallParser : Parser() {
override fun tryToParse(input: TokenList): ParserOutput {
return allOf(
SimpleIdentifierParser(),
FunctionCallArgumentsParser()
) {
FunctionCallNode(it[0], it[1])
}.parse(input)
}
}

View File

@@ -0,0 +1,13 @@
package io.smnp.dsl.ast.parser
import io.smnp.dsl.ast.model.entity.ParserOutput
import io.smnp.dsl.token.model.entity.TokenList
class FunctionDefinitionArgumentParser : Parser() {
override fun tryToParse(input: TokenList): ParserOutput {
return oneOf(
OptionalFunctionDefinitionArgumentParser(),
RegularFunctionDefinitionArgumentParser()
).parse(input)
}
}

View File

@@ -0,0 +1,12 @@
package io.smnp.dsl.ast.parser
import io.smnp.dsl.ast.model.node.FunctionDefinitionArgumentsNode
import io.smnp.dsl.token.model.enumeration.TokenType
class FunctionDefinitionArgumentsParser : AbstractIterableParser(
TokenType.OPEN_PAREN,
FunctionDefinitionArgumentParser(),
TokenType.CLOSE_PAREN,
{ list, tokenPosition ->
FunctionDefinitionArgumentsNode(list, tokenPosition)
})

View File

@@ -0,0 +1,19 @@
package io.smnp.dsl.ast.parser
import io.smnp.dsl.ast.model.entity.ParserOutput
import io.smnp.dsl.ast.model.node.FunctionDefinitionNode
import io.smnp.dsl.token.model.entity.TokenList
import io.smnp.dsl.token.model.enumeration.TokenType
class FunctionDefinitionParser : Parser() {
override fun tryToParse(input: TokenList): ParserOutput {
return allOf(
terminal(TokenType.FUNCTION),
assert(SimpleIdentifierParser(), "function/method name"),
assert(FunctionDefinitionArgumentsParser(), "function/method arguments list"),
assert(BlockParser(), "function/method body")
) {
FunctionDefinitionNode(it[1], it[2], it[3], it[0].position)
}.parse(input)
}
}

View File

@@ -0,0 +1,23 @@
package io.smnp.dsl.ast.parser
import io.smnp.dsl.ast.model.entity.ParserOutput
import io.smnp.dsl.ast.model.node.ImportNode
import io.smnp.dsl.token.model.entity.TokenList
import io.smnp.dsl.token.model.enumeration.TokenType
class ImportParser : Parser() {
override fun tryToParse(input: TokenList): ParserOutput {
val pathParser = oneOf(
UnitParser(),
SimpleIdentifierParser(),
StringLiteralParser()
)
return allOf(
terminal(TokenType.IMPORT),
pathParser
) {
ImportNode(it[1], it[0].position)
}.parse(input)
}
}

View File

@@ -0,0 +1,12 @@
package io.smnp.dsl.ast.parser
import io.smnp.dsl.ast.model.entity.ParserOutput
import io.smnp.dsl.ast.model.node.IntegerLiteralNode
import io.smnp.dsl.token.model.entity.TokenList
import io.smnp.dsl.token.model.enumeration.TokenType
class IntegerLiteralParser : Parser() {
override fun tryToParse(input: TokenList): ParserOutput {
return terminal(TokenType.INTEGER) { IntegerLiteralNode(it) }.parse(input)
}
}

View File

@@ -0,0 +1,12 @@
package io.smnp.dsl.ast.parser
import io.smnp.dsl.ast.model.node.ListNode
import io.smnp.dsl.token.model.enumeration.TokenType
class ListParser : AbstractIterableParser(
TokenType.OPEN_SQUARE,
SubexpressionParser(),
TokenType.CLOSE_SQUARE,
{ list, tokenPosition ->
ListNode(list, tokenPosition)
})

View File

@@ -0,0 +1,16 @@
package io.smnp.dsl.ast.parser
import io.smnp.dsl.ast.model.entity.ParserOutput
import io.smnp.dsl.token.model.entity.TokenList
class LiteralParser : Parser() {
override fun tryToParse(input: TokenList): ParserOutput {
return oneOf(
BoolLiteralParser(),
FloatLiteralParser(),
IntegerLiteralParser(),
NoteLiteralParser(),
StringLiteralParser()
).parse(input)
}
}

View File

@@ -0,0 +1,12 @@
package io.smnp.dsl.ast.parser
import io.smnp.dsl.ast.model.node.LoopParametersNode
import io.smnp.dsl.token.model.enumeration.TokenType
class LoopParametersParser : AbstractIterableParser(
TokenType.OPEN_PAREN,
SimpleIdentifierParser(),
TokenType.CLOSE_PAREN,
{ list, tokenPosition ->
LoopParametersNode(list, tokenPosition)
})

View File

@@ -0,0 +1,34 @@
package io.smnp.dsl.ast.parser
import io.smnp.dsl.ast.model.entity.ParserOutput
import io.smnp.dsl.ast.model.node.LoopNode
import io.smnp.dsl.ast.model.node.LoopParametersNode
import io.smnp.dsl.token.model.entity.TokenList
import io.smnp.dsl.token.model.enumeration.TokenType
class LoopParser : Parser() {
override fun tryToParse(input: TokenList): ParserOutput {
val loopParametersParser = allOf(
terminal(TokenType.AS),
oneOf(
mapNode(SimpleIdentifierParser()) { LoopParametersNode(listOf(it), it.position) },
LoopParametersParser()
)
) { it[1] }
val loopFilterParser = allOf(
terminal(TokenType.PERCENT),
assert(SubexpressionParser(), "filter as bool expression")
) { it[1] }
return allOf(
SubexpressionParser(),
optional(loopParametersParser),
terminal(TokenType.CARET),
StatementParser(),
optional(loopFilterParser)
) {
LoopNode(it[0], it[1], it[2], it[3], it[4])
}.parse(input)
}
}

View File

@@ -0,0 +1,23 @@
package io.smnp.dsl.ast.parser
import io.smnp.dsl.ast.model.entity.ParserOutput
import io.smnp.dsl.ast.model.node.MapEntryNode
import io.smnp.dsl.token.model.entity.TokenList
import io.smnp.dsl.token.model.enumeration.TokenType
class MapEntryParser : Parser() {
override fun tryToParse(input: TokenList): ParserOutput {
val keyParser = oneOf(
LiteralParser(),
SimpleIdentifierParser()
)
return allOf(
keyParser,
terminal(TokenType.ARROW),
assert(SubexpressionParser(), "expression")
) {
MapEntryNode(it[0], it[1], it[2])
}.parse(input)
}
}

View File

@@ -0,0 +1,9 @@
package io.smnp.dsl.ast.parser
import io.smnp.dsl.ast.model.node.MapNode
import io.smnp.dsl.token.model.enumeration.TokenType
class MapParser :
AbstractIterableParser(TokenType.OPEN_CURLY, MapEntryParser(), TokenType.CLOSE_CURLY, { list, tokenPosition ->
MapNode(list, tokenPosition)
})

View File

@@ -0,0 +1,12 @@
package io.smnp.dsl.ast.parser
import io.smnp.dsl.ast.model.entity.ParserOutput
import io.smnp.dsl.ast.model.node.NoteLiteralNode
import io.smnp.dsl.token.model.entity.TokenList
import io.smnp.dsl.token.model.enumeration.TokenType
class NoteLiteralParser : Parser() {
override fun tryToParse(input: TokenList): ParserOutput {
return terminal(TokenType.NOTE) { NoteLiteralNode(it) }.parse(input)
}
}

View File

@@ -0,0 +1,22 @@
package io.smnp.dsl.ast.parser
import io.smnp.dsl.ast.model.entity.ParserOutput
import io.smnp.dsl.ast.model.node.OptionalFunctionDefinitionArgumentNode
import io.smnp.dsl.token.model.entity.TokenList
import io.smnp.dsl.token.model.enumeration.TokenType
class OptionalFunctionDefinitionArgumentParser : Parser() {
override fun tryToParse(input: TokenList): ParserOutput {
return allOf(
SimpleIdentifierParser(),
optional(allOf(
terminal(TokenType.COLON),
TypeParser()
) { it[1] }),
terminal(TokenType.ASSIGN),
assert(ExpressionParser(), "expression")
) {
OptionalFunctionDefinitionArgumentNode(it[0], it[1], it[3])
}.parse(input)
}
}

View File

@@ -0,0 +1,208 @@
package io.smnp.dsl.ast.parser
import io.smnp.dsl.ast.model.entity.ParserOutput
import io.smnp.dsl.ast.model.enumeration.ParsingResult
import io.smnp.dsl.ast.model.node.Node
import io.smnp.dsl.ast.model.node.TokenNode
import io.smnp.dsl.token.model.entity.Token
import io.smnp.dsl.token.model.entity.TokenList
import io.smnp.dsl.token.model.entity.TokenPosition
import io.smnp.dsl.token.model.enumeration.TokenType
import io.smnp.error.InvalidSyntaxException
abstract class Parser {
fun parse(input: TokenList): ParserOutput {
val snapshot = input.snapshot()
val output = tryToParse(input)
if (output.result == ParsingResult.FAILED) {
input.restore(snapshot)
}
return output
}
protected abstract fun tryToParse(input: TokenList): ParserOutput
companion object {
// a -> A
fun terminal(terminal: TokenType, createNode: (Token) -> Node = { TokenNode(it) }): Parser {
return object : Parser() {
override fun tryToParse(input: TokenList): ParserOutput {
if (input.hasCurrent() && input.current.type == terminal) {
val token = input.current
input.ahead()
return ParserOutput.ok(createNode(token))
}
return ParserOutput.fail()
}
}
}
// oneOf -> a | b | c | ...
fun oneOf(vararg parsers: Parser): Parser {
return object : Parser() {
override fun tryToParse(input: TokenList): ParserOutput {
if (parsers.isEmpty()) {
throw RuntimeException("Provide one parser at least")
}
for (parser in parsers) {
val output = parser.parse(input)
if (output.result == ParsingResult.OK) {
return output
}
}
return ParserOutput.fail()
}
}
}
// a -> A | B | C | ...
fun terminals(vararg terminals: TokenType, createNode: (Token) -> Node = { TokenNode(it) }): Parser {
return oneOf(*terminals.map { terminal(it, createNode) }.toTypedArray())
}
// allOf -> a b c ...
fun allOf(vararg parsers: Parser, createNode: (List<Node>) -> Node): Parser {
return object : Parser() {
override fun tryToParse(input: TokenList): ParserOutput {
if (parsers.isEmpty()) {
throw RuntimeException("Provide one parser at least")
}
val nodes = mutableListOf<Node>()
for (parser in parsers) {
val output = parser.parse(input)
if (output.result == ParsingResult.FAILED) {
return ParserOutput.fail()
}
nodes += output.node
}
return ParserOutput.ok(createNode(nodes))
}
}
}
// leftAssociative -> left | left OP right
fun leftAssociativeOperator(
lhsParser: Parser,
allowedOperators: List<TokenType>,
rhsParser: Parser,
createNode: (lhs: Node, operator: Node, rhs: Node) -> Node
): Parser {
return object : Parser() {
override fun tryToParse(input: TokenList): ParserOutput {
val opParser = terminals(*allowedOperators.toTypedArray())
var lhs = lhsParser.parse(input)
if (lhs.result == ParsingResult.OK) {
var op = opParser.parse(input)
while (op.result == ParsingResult.OK) {
val rhs = rhsParser.parse(input)
if (rhs.result == ParsingResult.FAILED) { // Todo: Not sure if it should be here
return ParserOutput.fail()
}
lhs = ParserOutput.ok(createNode(lhs.node, op.node, rhs.node))
op = opParser.parse(input)
}
return lhs
}
return ParserOutput.fail()
}
}
}
// repeat -> item*
fun repeat(itemParser: Parser, createNode: (List<Node>, TokenPosition) -> Node): Parser {
return object : Parser() {
override fun tryToParse(input: TokenList): ParserOutput {
val items = mutableListOf<Node>()
while (true) {
val output = itemParser.parse(input)
if (output.result == ParsingResult.OK) {
items += output.node
} else if (items.isEmpty()) {
return ParserOutput.fail()
} else {
return ParserOutput.ok(createNode(items, items.first().position))
}
}
}
}
}
// loop -> begin item* end
fun loop(
beginParser: Parser,
itemParser: Parser,
endParser: Parser,
createNode: (Node, List<Node>, Node) -> Node
): Parser {
return object : Parser() {
override fun tryToParse(input: TokenList): ParserOutput {
val items = mutableListOf<Node>()
val begin = beginParser.parse(input)
if (begin.result == ParsingResult.OK) {
while (true) {
val end = endParser.parse(input)
if (end.result == ParsingResult.OK) {
return ParserOutput.ok(createNode(begin.node, items, end.node))
}
val item = itemParser.parse(input)
if (item.result == ParsingResult.FAILED) {
return ParserOutput.fail()
}
items += item.node
}
}
return ParserOutput.fail()
}
}
}
fun assert(parser: Parser, expected: String): Parser {
return object : Parser() {
override fun tryToParse(input: TokenList): ParserOutput {
val output = parser.parse(input)
if (output.result == ParsingResult.FAILED) {
throw InvalidSyntaxException("Expected $expected, got '${input.current.rawValue}'", input.currentPos())
}
return output
}
}
}
// optional -> a?
fun optional(parser: Parser): Parser {
return object : Parser() {
override fun tryToParse(input: TokenList): ParserOutput {
val output = parser.parse(input)
return if (output.result == ParsingResult.OK) output else ParserOutput.ok(Node.NONE)
}
}
}
fun mapNode(parser: Parser, mapper: (Node) -> Node): Parser {
return object : Parser() {
override fun tryToParse(input: TokenList): ParserOutput {
return parser.parse(input).map(mapper)
}
}
}
}
}

View File

@@ -0,0 +1,21 @@
package io.smnp.dsl.ast.parser
import io.smnp.dsl.ast.model.entity.ParserOutput
import io.smnp.dsl.ast.model.node.RegularFunctionDefinitionArgumentNode
import io.smnp.dsl.token.model.entity.TokenList
import io.smnp.dsl.token.model.enumeration.TokenType
class RegularFunctionDefinitionArgumentParser : Parser() {
override fun tryToParse(input: TokenList): ParserOutput {
return allOf(
optional(terminal(TokenType.DOTS)),
SimpleIdentifierParser(),
optional(allOf(
terminal(TokenType.COLON),
TypeParser()
) { it[1] })
) {
RegularFunctionDefinitionArgumentNode(it[1], it[2], it[0])
}.parse(input)
}
}

View File

@@ -0,0 +1,17 @@
package io.smnp.dsl.ast.parser
import io.smnp.dsl.ast.model.entity.ParserOutput
import io.smnp.dsl.ast.model.node.ReturnNode
import io.smnp.dsl.token.model.entity.TokenList
import io.smnp.dsl.token.model.enumeration.TokenType
class ReturnParser : Parser() {
override fun tryToParse(input: TokenList): ParserOutput {
return allOf(
terminal(TokenType.RETURN),
optional(ExpressionParser())
) {
ReturnNode(it[1])
}.parse(input)
}
}

View File

@@ -0,0 +1,20 @@
package io.smnp.dsl.ast.parser
import io.smnp.dsl.ast.model.entity.ParserOutput
import io.smnp.dsl.ast.model.node.RootNode
import io.smnp.dsl.token.model.entity.TokenList
class RootParser : Parser() {
override fun tryToParse(input: TokenList): ParserOutput {
return repeat(
oneOf(
ImportParser(),
FunctionDefinitionParser(),
ExtendParser(),
StatementParser()
)
) { list, tokenPosition ->
RootNode(list, tokenPosition)
}.parse(input)
}
}

View File

@@ -0,0 +1,16 @@
package io.smnp.dsl.ast.parser
import io.smnp.dsl.ast.model.entity.ParserOutput
import io.smnp.dsl.ast.model.node.IdentifierNode
import io.smnp.dsl.token.model.entity.TokenList
import io.smnp.dsl.token.model.enumeration.TokenType
class SimpleIdentifierParser : Parser() {
override fun tryToParse(input: TokenList): ParserOutput {
val identifierParser = terminal(TokenType.IDENTIFIER) { IdentifierNode(it) }
return oneOf(
identifierParser
).parse(input)
}
}

View File

@@ -0,0 +1,22 @@
package io.smnp.dsl.ast.parser
import io.smnp.dsl.ast.model.entity.ParserOutput
import io.smnp.dsl.token.model.entity.TokenList
import io.smnp.dsl.token.model.enumeration.TokenType
class StatementParser : Parser() {
override fun tryToParse(input: TokenList): ParserOutput {
return allOf(
oneOf(
ConditionParser(),
ExpressionParser(),
BlockParser(),
ReturnParser(),
ThrowParser()
),
optional(terminal(TokenType.SEMICOLON))
) {
it[0]
}.parse(input)
}
}

View File

@@ -0,0 +1,12 @@
package io.smnp.dsl.ast.parser
import io.smnp.dsl.ast.model.entity.ParserOutput
import io.smnp.dsl.ast.model.node.StringLiteralNode
import io.smnp.dsl.token.model.entity.TokenList
import io.smnp.dsl.token.model.enumeration.TokenType
class StringLiteralParser : Parser() {
override fun tryToParse(input: TokenList): ParserOutput {
return terminal(TokenType.STRING) { StringLiteralNode(it) }.parse(input)
}
}

View File

@@ -0,0 +1,46 @@
package io.smnp.dsl.ast.parser
import io.smnp.dsl.ast.model.entity.ParserOutput
import io.smnp.dsl.ast.model.node.LogicOperatorNode
import io.smnp.dsl.ast.model.node.RelationOperatorNode
import io.smnp.dsl.ast.model.node.SumOperatorNode
import io.smnp.dsl.token.model.entity.TokenList
import io.smnp.dsl.token.model.enumeration.TokenType
class SubexpressionParser : Parser() {
override fun tryToParse(input: TokenList): ParserOutput {
val expr1Parser = leftAssociativeOperator(
TermParser(),
listOf(TokenType.PLUS, TokenType.MINUS),
TermParser()
) { lhs, operator, rhs ->
SumOperatorNode(lhs, operator, rhs)
}
val expr2Parser = leftAssociativeOperator(
expr1Parser,
listOf(TokenType.AND),
expr1Parser
) { lhs, operator, rhs ->
LogicOperatorNode(lhs, operator, rhs)
}
val expr3Parser = leftAssociativeOperator(
expr2Parser,
listOf(TokenType.OR),
expr2Parser
) { lhs, operator, rhs ->
LogicOperatorNode(lhs, operator, rhs)
}
val expr4Parser = leftAssociativeOperator(
expr3Parser,
listOf(TokenType.RELATION, TokenType.OPEN_ANGLE, TokenType.CLOSE_ANGLE),
expr3Parser
) { lhs, operator, rhs ->
RelationOperatorNode(lhs, operator, rhs)
}
return expr4Parser.parse(input)
}
}

View File

@@ -0,0 +1,18 @@
package io.smnp.dsl.ast.parser
import io.smnp.dsl.ast.model.entity.ParserOutput
import io.smnp.dsl.ast.model.node.ProductOperatorNode
import io.smnp.dsl.token.model.entity.TokenList
import io.smnp.dsl.token.model.enumeration.TokenType
class TermParser : Parser() {
override fun tryToParse(input: TokenList): ParserOutput {
return leftAssociativeOperator(
FactorParser(),
listOf(TokenType.ASTERISK, TokenType.SLASH),
FactorParser()
) { lhs, operator, rhs ->
ProductOperatorNode(lhs, operator, rhs)
}.parse(input)
}
}

View File

@@ -0,0 +1,17 @@
package io.smnp.dsl.ast.parser
import io.smnp.dsl.ast.model.entity.ParserOutput
import io.smnp.dsl.ast.model.node.ThrowNode
import io.smnp.dsl.token.model.entity.TokenList
import io.smnp.dsl.token.model.enumeration.TokenType
class ThrowParser : Parser() {
override fun tryToParse(input: TokenList): ParserOutput {
return allOf(
terminal(TokenType.THROW),
optional(SubexpressionParser())
) {
ThrowNode(it[1])
}.parse(input)
}
}

View File

@@ -0,0 +1,17 @@
package io.smnp.dsl.ast.parser
import io.smnp.dsl.ast.model.entity.ParserOutput
import io.smnp.dsl.ast.model.node.TypeNode
import io.smnp.dsl.ast.model.node.TypeSpecifiersNode
import io.smnp.dsl.token.model.entity.TokenList
class TypeParser : Parser() {
override fun tryToParse(input: TokenList): ParserOutput {
return allOf(
SimpleIdentifierParser(),
optional(repeat(TypeSpecifierParser()) { list, tokenPosition -> TypeSpecifiersNode(list, tokenPosition) })
) {
TypeNode(it[0], it[1])
}.parse(input)
}
}

View File

@@ -0,0 +1,9 @@
package io.smnp.dsl.ast.parser
import io.smnp.dsl.ast.model.node.TypeSpecifierNode
import io.smnp.dsl.token.model.enumeration.TokenType
class TypeSpecifierParser :
AbstractIterableParser(TokenType.OPEN_ANGLE, TypeParser(), TokenType.CLOSE_ANGLE, { list, tokenPosition ->
TypeSpecifierNode(list, tokenPosition)
})

View File

@@ -0,0 +1,27 @@
package io.smnp.dsl.ast.parser
import io.smnp.dsl.ast.model.entity.ParserOutput
import io.smnp.dsl.ast.model.node.AccessOperatorNode
import io.smnp.dsl.ast.model.node.MinusOperatorNode
import io.smnp.dsl.token.model.entity.TokenList
import io.smnp.dsl.token.model.enumeration.TokenType
class UnitParser : Parser() {
override fun tryToParse(input: TokenList): ParserOutput {
val minusOperatorParser = allOf(
terminal(TokenType.MINUS),
assert(AtomParser(), "atom")
) {
MinusOperatorNode(it[0], it[1])
}
val atom2 = oneOf(
minusOperatorParser,
AtomParser()
)
return leftAssociativeOperator(atom2, listOf(TokenType.DOT), assert(atom2, "atom")) { lhs, operator, rhs ->
AccessOperatorNode(lhs, operator, rhs)
}.parse(input)
}
}

View File

@@ -0,0 +1,19 @@
package io.smnp.dsl.token.model.entity
import io.smnp.dsl.token.model.enumeration.TokenType
data class Token(val type: TokenType, val value: Any, val rawValue: String, val position: TokenPosition) {
constructor(type: TokenType, value: String, position: TokenPosition): this(type, value, value, position)
fun mapValue(mapper: (Any) -> Any): Token {
return Token(type, mapper(value), rawValue, position)
}
companion object {
val NONE = Token(TokenType.NONE, "", TokenPosition.NONE)
}
override fun toString(): String {
return "(${type.name.toLowerCase()}, »${rawValue}«, ${position.short()})"
}
}

View File

@@ -0,0 +1,60 @@
package io.smnp.dsl.token.model.entity
class TokenList(val tokens: List<Token>, val lines: List<String>) {
private var cursor = 0
private var snap = 0
operator fun get(index: Int): Token {
return tokens[index]
}
val current: Token
get() {
if (!hasCurrent()) {
throw RuntimeException("Cursor points to not existing token! Cursor = ${cursor}, length = ${tokens.size}")
}
return tokens[cursor]
}
fun currentPos(): TokenPosition {
return current.position
}
fun hasCurrent(): Boolean {
return cursor < tokens.size
}
fun hasMore(count: Int = 1): Boolean {
return cursor + count < tokens.size
}
fun next(number: Int = 1): Token {
return tokens[cursor + number]
}
fun prev(number: Int = 1): Token {
return tokens[cursor - number]
}
fun ahead() {
cursor += 1
}
fun snapshot(): Int {
return cursor
}
fun restore(snapshot: Int) {
cursor = snapshot
}
fun reset() {
cursor = 0
}
override fun toString(): String {
val currentStr = if(hasCurrent()) "current: ${cursor} -> ${current}" else "<all tokens consumed>"
return "size: ${tokens.size}\n${currentStr}\nall: ${tokens}"
}
}

View File

@@ -0,0 +1,15 @@
package io.smnp.dsl.token.model.entity
data class TokenPosition(val line: Int, val beginCol: Int, val endCol: Int) {
companion object {
val NONE = TokenPosition(-1, -1, -1)
}
override fun toString(): String {
return "[line ${line+1}, col ${beginCol}]"
}
fun short(): String {
return "${line+1}:${beginCol}"
}
}

View File

@@ -0,0 +1,29 @@
package io.smnp.dsl.token.model.entity
import io.smnp.dsl.token.model.enumeration.TokenType
data class TokenizerOutput(val consumedChars: Int, val token: Token) {
fun consumed(): Boolean {
return consumedChars > 0
}
fun mapToken(mapper: (Token) -> Token): TokenizerOutput {
return TokenizerOutput(consumedChars, mapper(token) )
}
companion object {
val NONE = TokenizerOutput(0, Token.NONE)
fun produce(consumedChars: Int, token: Token): TokenizerOutput {
return if (consumedChars > 0) TokenizerOutput(consumedChars, token) else NONE
}
fun produce(consumedChars: Int, value: String, tokenType: TokenType, line: Int, beginCol: Int): TokenizerOutput {
return produce(consumedChars, Token(tokenType, value, TokenPosition(line, beginCol, beginCol + consumedChars)))
}
fun produce(consumedChars: Int, value: Any, rawValue: String, tokenType: TokenType, line: Int, beginCol: Int): TokenizerOutput {
return produce(consumedChars, Token(tokenType, value, rawValue, TokenPosition(line, beginCol, beginCol + consumedChars)))
}
}
}

View File

@@ -0,0 +1,50 @@
package io.smnp.dsl.token.model.enumeration
enum class TokenType(val token: String) {
NONE("<NONE>"),
RELATION("==, !=, >=, <="),
DOUBLE_ASTERISK("**"),
OPEN_CURLY("{"),
CLOSE_CURLY("}"),
OPEN_PAREN("("),
CLOSE_PAREN(")"),
OPEN_SQUARE("["),
CLOSE_SQUARE("]"),
OPEN_ANGLE("<"),
CLOSE_ANGLE(">"),
SEMICOLON(";"),
ASTERISK("*"),
PERCENT("%"),
ASSIGN("="),
ARROW("->"),
COLON(":"),
COMMA(","),
SLASH("/"),
MINUS("-"),
PLUS("+"),
CARET("^"),
DOTS("..."),
AMP("&"),
DOT("."),
AND("and"),
OR("or"),
NOT("not"),
INTEGER("integer"),
STRING("string"),
FLOAT("float"),
NOTE("note"),
BOOL("bool"),
TYPE("type"),
FUNCTION("function"),
RETURN("return"),
EXTEND("extend"),
IMPORT("import"),
THROW("throw"),
FROM("from"),
WITH("with"),
ELSE("else"),
IF("if"),
AS("as"),
IDENTIFIER("identifier"),
COMMENT("comment")
}

View File

@@ -0,0 +1,21 @@
package io.smnp.dsl.token.tokenizer
import io.smnp.dsl.token.model.entity.TokenizerOutput
import io.smnp.dsl.token.model.enumeration.TokenType
class CommentTokenizer : Tokenizer {
override fun tokenize(input: String, current: Int, line: Int): TokenizerOutput {
if (input[current] == '#') {
var consumedChars = 0
var value = ""
while(current + consumedChars < input.length) {
value += input[current + consumedChars]
consumedChars += 1
}
return TokenizerOutput.produce(consumedChars, value.substring(1).trim(), value, TokenType.COMMENT, line, current)
}
return TokenizerOutput.NONE
}
}

View File

@@ -0,0 +1,113 @@
package io.smnp.dsl.token.tokenizer
import io.smnp.dsl.token.model.entity.Token
import io.smnp.dsl.token.model.entity.TokenList
import io.smnp.dsl.token.model.entity.TokenPosition
import io.smnp.dsl.token.model.entity.TokenizerOutput
import io.smnp.dsl.token.model.enumeration.TokenType
import io.smnp.dsl.token.tokenizer.Tokenizer.Companion.default
import io.smnp.dsl.token.tokenizer.Tokenizer.Companion.keywords
import io.smnp.dsl.token.tokenizer.Tokenizer.Companion.mapValue
import io.smnp.dsl.token.tokenizer.Tokenizer.Companion.regex
import io.smnp.dsl.token.tokenizer.Tokenizer.Companion.separated
import io.smnp.error.InvalidSyntaxException
class DefaultTokenizer : Tokenizer {
private val tokenizers = listOf(
default(TokenType.ARROW),
// Double-character operators
keywords(TokenType.RELATION, "==", "!=", "<=", ">="),
default(TokenType.DOUBLE_ASTERISK),
// Characters
default(TokenType.OPEN_CURLY),
default(TokenType.CLOSE_CURLY),
default(TokenType.OPEN_PAREN),
default(TokenType.CLOSE_PAREN),
default(TokenType.OPEN_SQUARE),
default(TokenType.CLOSE_SQUARE),
default(TokenType.OPEN_ANGLE),
default(TokenType.CLOSE_ANGLE),
default(TokenType.SEMICOLON),
default(TokenType.ASTERISK),
default(TokenType.PERCENT),
default(TokenType.ASSIGN),
default(TokenType.COLON),
default(TokenType.COMMA),
default(TokenType.SLASH),
default(TokenType.MINUS),
default(TokenType.PLUS),
default(TokenType.CARET),
default(TokenType.DOTS),
default(TokenType.AMP),
default(TokenType.DOT),
// Types
mapValue(separated(FloatTokenizer())) { (it as String).toFloat() },
mapValue(separated(regex(TokenType.INTEGER, "\\d"))) { (it as String).toInt() },
StringTokenizer(),
separated(NoteTokenizer()),
mapValue(separated(keywords(TokenType.BOOL, "true", "false"))) { it == "true" },
// Keywords
separated(default(TokenType.FUNCTION)),
separated(default(TokenType.RETURN)),
separated(default(TokenType.EXTEND)),
separated(default(TokenType.IMPORT)),
separated(default(TokenType.THROW)),
separated(default(TokenType.FROM)),
separated(default(TokenType.WITH)),
separated(default(TokenType.ELSE)),
separated(default(TokenType.AND)),
separated(default(TokenType.NOT)),
separated(default(TokenType.AS)),
separated(default(TokenType.IF)),
separated(default(TokenType.OR)),
// Identifier (couldn't be before keywords!)
regex(TokenType.IDENTIFIER, "\\w"),
// Whitespaces
regex(TokenType.NONE, "\\s"),
// Comments
CommentTokenizer()
)
private val filters: List<(Token) -> Boolean> = listOf(
{ token -> token.type != TokenType.NONE },
{ token -> token.type != TokenType.COMMENT }
)
private val tokenizer = Tokenizer.firstOf(tokenizers)
fun tokenize(lines: List<String>): TokenList {
val tokens: MutableList<Token> = mutableListOf()
for ((index, line) in lines.withIndex()) {
var current = 0
while (current < line.length) {
val output = tokenize(line, current, index)
if (!output.consumed()) {
throw InvalidSyntaxException("Unknown symbol ${line[current]}", TokenPosition(index, current, -1))
}
current += output.consumedChars
tokens.add(output.token)
}
}
val filteredTokens = filterTokens(tokens.toList())
return TokenList(filteredTokens, lines)
}
private fun filterTokens(tokens: List<Token>): List<Token> {
return tokens.filter { token -> filters.all { filter -> filter(token) } }
}
override fun tokenize(input: String, current: Int, line: Int): TokenizerOutput {
return tokenizer.tokenize(input, current, line)
}
}

View File

@@ -0,0 +1,16 @@
package io.smnp.dsl.token.tokenizer
import io.smnp.dsl.token.model.entity.Token
import io.smnp.dsl.token.model.entity.TokenizerOutput
import io.smnp.dsl.token.model.enumeration.TokenType
class FloatTokenizer : Tokenizer {
override fun tokenize(input: String, current: Int, line: Int): TokenizerOutput {
return Tokenizer.combined(
{ values, tokenPosition -> Token(TokenType.FLOAT, values.joinToString(""), tokenPosition) },
Tokenizer.regex(TokenType.NONE, "\\d"),
Tokenizer.keyword(TokenType.NONE, "."),
Tokenizer.regex(TokenType.NONE, "\\d")
).tokenize(input, current, line)
}
}

View File

@@ -0,0 +1,76 @@
package io.smnp.dsl.token.tokenizer
import io.smnp.data.entity.Note
import io.smnp.data.enumeration.Pitch
import io.smnp.dsl.token.model.entity.TokenizerOutput
import io.smnp.dsl.token.model.enumeration.TokenType
class NoteTokenizer : Tokenizer {
override fun tokenize(input: String, current: Int, line: Int): TokenizerOutput {
var consumedChars = 0
var notePitch: String
var octave: Int? = null
var duration: String? = null
var dot = false
var rawValue = ""
// Note literal start symbol
if(input[current] == '@') {
rawValue += input[current+consumedChars]
consumedChars += 1
// Note basic pitch
if(listOf('c', 'd', 'e', 'f', 'g', 'a', 'h', 'b').contains(input[current+consumedChars].toLowerCase())) {
rawValue += input[current+consumedChars]
notePitch = input[current+consumedChars].toString()
consumedChars += 1
// Flat or sharp
if(current+consumedChars < input.length && listOf('b', '#').contains(input[current+consumedChars])) {
rawValue += input[current+consumedChars]
notePitch += input[current+consumedChars]
consumedChars += 1
}
// Octave
if(current+consumedChars < input.length && "\\d".toRegex().matches(input[current+consumedChars].toString())) {
rawValue += input[current+consumedChars]
octave = input[current+consumedChars].toString().toInt()
consumedChars += 1
}
// Duration start symbol
if(current+consumedChars < input.length && input[current+consumedChars] == ':') {
rawValue += input[current+consumedChars]
duration = ""
consumedChars += 1
while(current+consumedChars < input.length && "\\d".toRegex().matches(input[current+consumedChars].toString())) {
rawValue += input[current+consumedChars]
duration += input[current+consumedChars]
consumedChars += 1
}
if(duration.isEmpty()) {
return TokenizerOutput.NONE
}
dot = (current+consumedChars < input.length && input[current+consumedChars] == 'd')
if(dot) {
rawValue += input[current+consumedChars]
consumedChars += 1
}
}
val note = Note.Builder()
.pitch(Pitch.parse(notePitch))
.dot(dot)
octave?.let { note.octave(it) }
duration?.let { note.duration(it.toInt()) }
return TokenizerOutput.produce(consumedChars, note.build(), rawValue, TokenType.NOTE, line, current)
}
}
return TokenizerOutput.NONE
}
}

View File

@@ -0,0 +1,24 @@
package io.smnp.dsl.token.tokenizer
import io.smnp.dsl.token.model.entity.TokenizerOutput
import io.smnp.dsl.token.model.enumeration.TokenType
class StringTokenizer : Tokenizer {
override fun tokenize(input: String, current: Int, line: Int): TokenizerOutput {
if(input[current] == '"') {
var value = input[current].toString()
var consumedChars = 1
while(input.length > current + consumedChars && input[current+consumedChars] != '"') {
value += input[current + consumedChars]
consumedChars += 1
}
value += input[current + consumedChars]
consumedChars += 1
return TokenizerOutput.produce(consumedChars, value.substring(1, value.length-1), value, TokenType.STRING, line, current)
}
return TokenizerOutput.NONE
}
}

View File

@@ -0,0 +1,152 @@
package io.smnp.dsl.token.tokenizer
import io.smnp.dsl.token.model.entity.Token
import io.smnp.dsl.token.model.entity.TokenPosition
import io.smnp.dsl.token.model.entity.TokenizerOutput
import io.smnp.dsl.token.model.enumeration.TokenType
interface Tokenizer {
fun tokenize(input: String, current: Int, line: Int): TokenizerOutput
companion object {
// Char regex
fun regex(type: TokenType, pattern: String): Tokenizer {
return object : Tokenizer {
override fun tokenize(input: String, current: Int, line: Int): TokenizerOutput {
var consumedChars = 0
var value = ""
while (current + consumedChars < input.length && pattern.toRegex().matches(input[current + consumedChars].toString())) {
value += input[current + consumedChars]
consumedChars += 1
}
return TokenizerOutput.produce(consumedChars, value, type, line, current)
}
}
}
// Literal keyword ("function", "or", ".")
fun keyword(type: TokenType, keyword: String): Tokenizer {
return object : Tokenizer {
override fun tokenize(input: String, current: Int, line: Int): TokenizerOutput {
if ((input.length >= current + keyword.length) && (input.substring(
current,
current + keyword.length
) == keyword)
) {
return TokenizerOutput.produce(keyword.length, keyword, type, line, current)
}
return TokenizerOutput.NONE
}
}
}
// One of keywords
fun keywords(type: TokenType, vararg keywords: String): Tokenizer {
return object : Tokenizer {
override fun tokenize(input: String, current: Int, line: Int): TokenizerOutput {
for (keyword in keywords) {
val output = keyword(type, keyword).tokenize(input, current, line)
if (output.consumed()) {
return output
}
}
return TokenizerOutput.NONE
}
}
}
// Token for regular TokenType
fun default(type: TokenType): Tokenizer {
return object : Tokenizer {
override fun tokenize(input: String, current: Int, line: Int): TokenizerOutput {
return keyword(type, type.token).tokenize(input, current, line)
}
}
}
// Isolate io.io.smnp.dsl.token (for example "function" | "functions" | "function s")
fun separated(tokenizer: Tokenizer, end: String = "\\W"): Tokenizer {
return object : Tokenizer {
override fun tokenize(input: String, current: Int, line: Int): TokenizerOutput {
val output = tokenizer.tokenize(input, current, line)
if (output.consumed()) {
if ((input.length > current + output.consumedChars) && end.toRegex().matches(input[current + output.consumedChars].toString())) {
return output
}
if ((input.length == current + output.consumedChars)) {
return output
}
}
return TokenizerOutput.NONE
}
}
}
// Change io.io.smnp.dsl.token value (rawValue will be kept)
fun mapValue(tokenizer: Tokenizer, mapper: (Any) -> Any): Tokenizer {
return object : Tokenizer {
override fun tokenize(input: String, current: Int, line: Int): TokenizerOutput {
val output = tokenizer.tokenize(input, current, line)
if(output.consumed()) {
return output.mapToken { output.token.mapValue(mapper) }
}
return TokenizerOutput.NONE
}
}
}
// Complex tokenizer consisting of smaller ones (like "3.14" = regex(\d) + keyword(.) + regex(\d))
fun combined(createToken: (List<String>, TokenPosition) -> Token, vararg tokenizers: Tokenizer): Tokenizer {
return object : Tokenizer {
override fun tokenize(input: String, current: Int, line: Int): TokenizerOutput {
var consumedChars = 0
val tokens: MutableList<Token> = mutableListOf()
for(tokenizer in tokenizers) {
val output = tokenizer.tokenize(input, current + consumedChars, line)
if(output.consumed()) {
consumedChars += output.consumedChars
tokens.add(output.token)
} else {
return TokenizerOutput.NONE
}
}
if(consumedChars == 0) {
return TokenizerOutput.NONE
}
return TokenizerOutput.produce(consumedChars, createToken(tokens.map { it.rawValue }, TokenPosition(line, tokens.first().position.beginCol, tokens.last().position.endCol)))
}
}
}
// First matched tokenizer
fun firstOf(tokenizers: List<Tokenizer>): Tokenizer {
return object : Tokenizer {
override fun tokenize(input: String, current: Int, line: Int): TokenizerOutput {
for (tokenizer in tokenizers) {
val output = tokenizer.tokenize(input, current, line)
if(output.consumed()) {
return output
}
}
return TokenizerOutput.NONE
}
}
}
}
}

View File

@@ -0,0 +1,8 @@
package io.smnp.error
import io.smnp.dsl.token.model.entity.TokenPosition
class EvaluationException(message: String?, val position: TokenPosition?) : Exception(message) {
override val message: String?
get() = super.message + if(position != null) " $position" else ""
}

View File

@@ -0,0 +1,8 @@
package io.smnp.error
import io.smnp.dsl.token.model.entity.TokenPosition
class InvalidSyntaxException(message: String?, val position: TokenPosition?) : Exception(message) {
override val message: String?
get() = super.message + if(position != null) " $position" else ""
}

View File

@@ -0,0 +1,33 @@
package io.smnp.evaluation.evaluator
import io.smnp.dsl.ast.model.node.AccessOperatorNode
import io.smnp.dsl.ast.model.node.FunctionCallNode
import io.smnp.dsl.ast.model.node.IdentifierNode
import io.smnp.dsl.ast.model.node.Node
import io.smnp.environment.Environment
import io.smnp.error.EvaluationException
import io.smnp.evaluation.model.entity.EvaluatorOutput
class AccessOperatorEvaluator : Evaluator() {
override fun supportedNodes() = listOf(AccessOperatorNode::class)
override fun tryToEvaluate(node: Node, environment: Environment): EvaluatorOutput {
val evaluator = ExpressionEvaluator()
val (lhsNode, _, rhsNode) = (node as AccessOperatorNode)
val lhs = evaluator.evaluate(lhsNode, environment).value!!
return when (rhsNode) {
is IdentifierNode -> {
val rhs = rhsNode.token.rawValue
EvaluatorOutput.value(lhs.properties[rhs] ?: throw EvaluationException("Unknown property $rhs of type ${lhs.type.name.toLowerCase()}", rhsNode.position))
}
is FunctionCallNode -> {
// todo Implement when methods become available
EvaluatorOutput.fail()
}
else -> {
throw EvaluationException("Invalid property access type - only property name and method call are allowed", rhsNode.position)
}
}
}
}

View File

@@ -0,0 +1,20 @@
package io.smnp.evaluation.evaluator
import io.smnp.dsl.ast.model.node.BlockNode
import io.smnp.dsl.ast.model.node.Node
import io.smnp.environment.Environment
import io.smnp.evaluation.model.entity.EvaluatorOutput
import io.smnp.evaluation.model.enumeration.EvaluationResult
class BlockEvaluator : Evaluator() {
override fun supportedNodes() = listOf(BlockNode::class)
override fun tryToEvaluate(node: Node, environment: Environment): EvaluatorOutput {
val evaluator = DefaultEvaluator()
val ok = (node as BlockNode).statements.all {
evaluator.evaluate(it, environment).result == EvaluationResult.OK
}
return if(ok) EvaluatorOutput.ok() else EvaluatorOutput.fail()
}
}

Some files were not shown because too many files have changed in this diff Show More