Introduce new tokens: $, | and ||

app/src/main/kotlin/io/smnp/dsl/token/model/enumeration/TokenType.kt

@@ -4,6 +4,7 @@ enum class TokenType(val token: String) {
     NONE("<NONE>"),
     RELATION("==, !=, >=, <="),
     DOUBLE_ASTERISK("**"),
+    DOUBLE_PIPE("||"),
     OPEN_CURLY("{"),
     CLOSE_CURLY("}"),
     OPEN_PAREN("("),
@@ -15,6 +16,7 @@ enum class TokenType(val token: String) {
     SEMICOLON(";"),
     ASTERISK("*"),
     PERCENT("%"),
+    DOLLAR("$"),
     ASSIGN("="),
     ARROW("->"),
     COLON(":"),
@@ -24,6 +26,7 @@ enum class TokenType(val token: String) {
     PLUS("+"),
     CARET("^"),
     DOTS("..."),
+    PIPE("|"),
     AMP("&"),
     DOT("."),
     AND("and"),

app/src/main/kotlin/io/smnp/dsl/token/tokenizer/DefaultTokenizer.kt

@@ -21,6 +21,9 @@ class DefaultTokenizer : Tokenizer {
         keywords(TokenType.RELATION, "==", "!=", "<=", ">="),
         default(TokenType.DOUBLE_ASTERISK),
 
+        // Double-character tokens
+        default(TokenType.DOUBLE_PIPE),
+
         // Characters
         default(TokenType.OPEN_CURLY),
         default(TokenType.CLOSE_CURLY),
@@ -33,6 +36,7 @@ class DefaultTokenizer : Tokenizer {
         default(TokenType.SEMICOLON),
         default(TokenType.ASTERISK),
         default(TokenType.PERCENT),
+        default(TokenType.DOLLAR),
         default(TokenType.ASSIGN),
         default(TokenType.COLON),
         default(TokenType.COMMA),
@@ -41,6 +45,7 @@ class DefaultTokenizer : Tokenizer {
         default(TokenType.PLUS),
         default(TokenType.CARET),
         default(TokenType.DOTS),
+        default(TokenType.PIPE),
         default(TokenType.AMP),
         default(TokenType.DOT),
 

app/src/main/kotlin/io/smnp/interpreter/DefaultInterpreter.kt

@@ -35,9 +35,9 @@ class DefaultInterpreter  {
       environment.loadModule("smnp.lang")
 
       val tokens = tokenizer.tokenize(lines, source)
-      val ast = parser.parse(tokens)
-
       if (printTokens) println(tokens)
+
+      val ast = parser.parse(tokens)
       if (printAst) ast.node.pretty()
 
       if (!dryRun) {