bartek/mvm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Create colon tokenizer

Browse Source
This commit is contained in:
Bartłomiej Przemysław Pluta
2021-11-03 19:12:08 +01:00
parent 8ae464047e
commit e0953a391b
1 changed files with 9 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
app/Assembler.hs
View File

@@ -8,9 +8,9 @@ import qualified VirtualMachine as VM (Op(..), Instruction, Command, instruction
import qualified Util as U import qualified Util as U
data Token = Operator VM.Op data Token = Operator VM.Op
| KeywordLiteral String
| IntLiteral Int | IntLiteral Int
| StringLiteral String | StringLiteral String
| Colon
| WhiteSpace | WhiteSpace
| Comment String | Comment String
deriving (Eq, Show) deriving (Eq, Show)
@@ -21,10 +21,10 @@ data TokenizeResult = TokenizeResult Token ConsumedChars deriving (Eq, Show)
type Tokenizer = String -> Maybe TokenizeResult type Tokenizer = String -> Maybe TokenizeResult
type CaseSensitive = Bool type CaseSensitive = Bool
tokenizeKeyword :: CaseSensitive -> String -> Tokenizer keywordTokenizer :: CaseSensitive -> String -> Token -> Tokenizer
tokenizeKeyword _ _ [] = Nothing keywordTokenizer _ _ _ [] = Nothing
tokenizeKeyword cs kwd input keywordTokenizer cs kwd token input
| matches = Just $ TokenizeResult (KeywordLiteral . take len $ input) len | matches = Just $ TokenizeResult token len
| otherwise = Nothing | otherwise = Nothing
where where
len = length kwd len = length kwd
@@ -32,14 +32,11 @@ tokenizeKeyword cs kwd input
zipped = zipWith (==) (mapper kwd) (mapper . take len $ input) zipped = zipWith (==) (mapper kwd) (mapper . take len $ input)
matches = and zipped && len == length zipped matches = and zipped && len == length zipped
tokenizeOperator :: VM.Op -> Tokenizer operatorTokenizer :: VM.Op -> Tokenizer
tokenizeOperator op input = case keywordToken of operatorTokenizer op input = keywordTokenizer False (U.toLowerCase . show $ op) (Operator op) input
(Just (TokenizeResult _ consumed)) -> Just $ TokenizeResult (Operator op) consumed
Nothing -> Nothing
where keywordToken = tokenizeKeyword False (U.toLowerCase . show $ op) input
tokenizeOperators :: Tokenizer tokenizeOperators :: Tokenizer
tokenizeOperators = anyTokenizer $ map tokenizeOperator [VM.Push ..] tokenizeOperators = anyTokenizer $ map operatorTokenizer [VM.Push ..]
tokenizeWhitespace :: Tokenizer tokenizeWhitespace :: Tokenizer
tokenizeWhitespace [] = Nothing tokenizeWhitespace [] = Nothing
@@ -125,6 +122,7 @@ tokenizers = anyTokenizer
, sepTokenizer Char.isSpace tokenizeOperators , sepTokenizer Char.isSpace tokenizeOperators
, sepTokenizer Char.isSpace tokenizeHex , sepTokenizer Char.isSpace tokenizeHex
, sepTokenizer Char.isSpace tokenizeDecimal , sepTokenizer Char.isSpace tokenizeDecimal
, keywordTokenizer False ":" Colon
, tokenizeChar , tokenizeChar
, tokenizeString , tokenizeString
] ]