Create Assembler module and implement some basic tokenizers

MVM.cabal

@@ -26,6 +26,7 @@ executable MVM
     -- Modules included in this executable, other than Main.
     other-modules:
       VirtualMachine
+      Assembler
       Util
 
     -- LANGUAGE extensions used by modules in this package.

app/Assembler.hs

@@ -0,0 +1,41 @@
+module Assembler (
+  tokenize
+) where
+
+import Data.Monoid as Monoid
+import qualified VirtualMachine as VM (Op(..), Instruction, Command, instructionByOp)
+import qualified Util as U
+
+data Token = Operator VM.Op | KeywordLiteral String | IntLiteral Int deriving (Eq, Show)
+
+type ConsumedChars = Int
+data TokenizeResult = TokenizeResult Token ConsumedChars deriving (Eq, Show)
+
+type Tokenizer = String -> Maybe TokenizeResult
+
+type CaseSensitive = Bool
+tokenizeKeyword :: CaseSensitive -> String -> Tokenizer
+tokenizeKeyword _ _ [] = Nothing
+tokenizeKeyword cs kwd input
+  | matches    = Just $ TokenizeResult (KeywordLiteral . take len $ input) len
+  | otherwise  = Nothing
+  where
+    len = length kwd
+    mapper = if cs then id else U.toLowerCase
+    zipped = zipWith (==) (mapper kwd) (mapper . take len $ input)
+    matches = and zipped && len == length zipped
+
+tokenizeOperator :: VM.Op -> Tokenizer
+tokenizeOperator op input = case keywordToken of
+  (Just (TokenizeResult _ consumed)) -> Just $ TokenizeResult (Operator op) consumed
+  Nothing                            -> Nothing
+  where keywordToken = tokenizeKeyword False (U.toLowerCase . show $ op) input
+
+anyTokenizer :: [Tokenizer] -> Tokenizer
+anyTokenizer tokenizers input = Monoid.getFirst . Monoid.mconcat . map Monoid.First $ sequenceA tokenizers input
+
+tokenizeOperators :: Tokenizer
+tokenizeOperators = anyTokenizer $ map tokenizeOperator [VM.Push ..]
+
+tokenize :: Tokenizer
+tokenize = tokenizeOperators

app/Util.hs

@@ -1,4 +1,5 @@
 module Util (
+  toLowerCase,
   byteStr,
   bytesStr
 ) where
@@ -6,6 +7,10 @@ module Util (
 import Data.List
 import Data.Word
 import Numeric (showHex)
+import qualified Data.Char as Char
+
+toLowerCase :: String -> String
+toLowerCase = map Char.toLower
 
 bytesStr :: Int -> [Word8] -> String
 bytesStr sparse = insertAtN '\n' (sparse*3) . intercalate " " . map byteStr