Enable splitting emitted tokens by new lines

app/Assembler.hs

@@ -2,8 +2,8 @@ module Assembler (
   tokenize
 ) where
 
-import Data.Char as Char
+import qualified Data.Char as Char
-import Data.Monoid as Monoid
+import qualified Data.Monoid as Monoid
 import qualified VirtualMachine as VM (Op(..), Instruction, Command, instructionByOp)
 import qualified Util as U
 
@@ -77,7 +77,7 @@ tokenizeHex input = if isPrefix && len > 0
 
 tokenizeChar :: Tokenizer
 tokenizeChar ('\'':'\\':x:'\'':_) = U.controlChar x >>= (\s -> return $ TokenizeResult (IntLiteral s) 4)
-tokenizeChar ('\'':x:'\'':_) = Just $ TokenizeResult (IntLiteral . ord $ x) 3      
+tokenizeChar ('\'':x:'\'':_) = Just $ TokenizeResult (IntLiteral . Char.ord $ x) 3      
 tokenizeChar _ = Nothing
 
 tokenizeString :: Tokenizer
@@ -121,10 +121,17 @@ anyTokenizer tokenizers input = Monoid.getFirst . Monoid.mconcat . map Monoid.Fi
 
 tokenize :: String -> Either String [Token]
 tokenize [] = Right []
-tokenize input = case tokenizers input of
+tokenize input = tokens >>= (\t -> Right $ filter tokenFilter t)
+  where
+    tokens = case tokenizers input of
       (Just (TokenizeResult token chars)) -> tokenize (drop chars input) >>= (\rest -> return $ token : rest)
       Nothing                             -> Left $ "Unknown token: " ++ take 20 input
 
+tokenFilter :: Token -> Bool
+tokenFilter (WhiteSpace) = False
+tokenFilter (Comment _)  = False
+tokenFilter _            = True
+
 tokenizers :: Tokenizer
 tokenizers = anyTokenizer
   [ keywordTokenizer False "\n" NewLine

app/Util.hs

@@ -4,7 +4,8 @@ module Util (
   bytesStr,
   head,
   unescape,
-  controlChar
+  controlChar,
+  explode
 ) where
 
 import Prelude hiding (head)
@@ -58,3 +59,10 @@ controlChar x = case x of
     '\'' -> Just 39
     '0'  -> Just 0
     _    -> Nothing
+
+explode :: (Foldable f) => (a -> Bool) -> f a -> [[a]]
+explode pred xs = filter (not . null) $ foldr split [[]] xs 
+  where 
+    split y (ys:yss) 
+      | pred y    = []:ys:yss
+      | otherwise = (y:ys):yss