bartek/mvm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Create new line tokenizer

Browse Source
This commit is contained in:
Bartłomiej Przemysław Pluta
2021-11-03 20:25:25 +01:00
parent 4a8a0ec437
commit 674c574fda
1 changed files with 3 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
app/Assembler.hs
View File

@@ -13,6 +13,7 @@ data Token = Operator VM.Op
| Identifier String | Identifier String
| Colon | Colon
| Ampersand | Ampersand
| NewLine
| WhiteSpace | WhiteSpace
| Comment String | Comment String
deriving (Eq, Show) deriving (Eq, Show)
@@ -126,7 +127,8 @@ tokenize input = case tokenizers input of
tokenizers :: Tokenizer tokenizers :: Tokenizer
tokenizers = anyTokenizer tokenizers = anyTokenizer
[ tokenizeWhitespace [ keywordTokenizer False "\n" NewLine
, tokenizeWhitespace
, tokenizeComment , tokenizeComment
, sepTokenizer Char.isSpace tokenizeOperators , sepTokenizer Char.isSpace tokenizeOperators
, sepTokenizer Char.isSpace tokenizeHex , sepTokenizer Char.isSpace tokenizeHex