diff --git a/app/src/main/kotlin/io/smnp/dsl/token/model/enumeration/TokenType.kt b/app/src/main/kotlin/io/smnp/dsl/token/model/enumeration/TokenType.kt index acbb138..0938f75 100644 --- a/app/src/main/kotlin/io/smnp/dsl/token/model/enumeration/TokenType.kt +++ b/app/src/main/kotlin/io/smnp/dsl/token/model/enumeration/TokenType.kt @@ -4,6 +4,7 @@ enum class TokenType(val token: String) { NONE(""), RELATION("==, !=, >=, <="), DOUBLE_ASTERISK("**"), + DOUBLE_PIPE("||"), OPEN_CURLY("{"), CLOSE_CURLY("}"), OPEN_PAREN("("), @@ -15,6 +16,7 @@ enum class TokenType(val token: String) { SEMICOLON(";"), ASTERISK("*"), PERCENT("%"), + DOLLAR("$"), ASSIGN("="), ARROW("->"), COLON(":"), @@ -24,6 +26,7 @@ enum class TokenType(val token: String) { PLUS("+"), CARET("^"), DOTS("..."), + PIPE("|"), AMP("&"), DOT("."), AND("and"), diff --git a/app/src/main/kotlin/io/smnp/dsl/token/tokenizer/DefaultTokenizer.kt b/app/src/main/kotlin/io/smnp/dsl/token/tokenizer/DefaultTokenizer.kt index cb204b9..5e76048 100644 --- a/app/src/main/kotlin/io/smnp/dsl/token/tokenizer/DefaultTokenizer.kt +++ b/app/src/main/kotlin/io/smnp/dsl/token/tokenizer/DefaultTokenizer.kt @@ -21,6 +21,9 @@ class DefaultTokenizer : Tokenizer { keywords(TokenType.RELATION, "==", "!=", "<=", ">="), default(TokenType.DOUBLE_ASTERISK), + // Double-character tokens + default(TokenType.DOUBLE_PIPE), + // Characters default(TokenType.OPEN_CURLY), default(TokenType.CLOSE_CURLY), @@ -33,6 +36,7 @@ class DefaultTokenizer : Tokenizer { default(TokenType.SEMICOLON), default(TokenType.ASTERISK), default(TokenType.PERCENT), + default(TokenType.DOLLAR), default(TokenType.ASSIGN), default(TokenType.COLON), default(TokenType.COMMA), @@ -41,6 +45,7 @@ class DefaultTokenizer : Tokenizer { default(TokenType.PLUS), default(TokenType.CARET), default(TokenType.DOTS), + default(TokenType.PIPE), default(TokenType.AMP), default(TokenType.DOT), diff --git a/app/src/main/kotlin/io/smnp/interpreter/DefaultInterpreter.kt b/app/src/main/kotlin/io/smnp/interpreter/DefaultInterpreter.kt index 2ec1f05..1042055 100644 --- a/app/src/main/kotlin/io/smnp/interpreter/DefaultInterpreter.kt +++ b/app/src/main/kotlin/io/smnp/interpreter/DefaultInterpreter.kt @@ -35,9 +35,9 @@ class DefaultInterpreter { environment.loadModule("smnp.lang") val tokens = tokenizer.tokenize(lines, source) - val ast = parser.parse(tokens) - if (printTokens) println(tokens) + + val ast = parser.parse(tokens) if (printAst) ast.node.pretty() if (!dryRun) {