Create char tokenizer
This commit is contained in:
@@ -67,6 +67,24 @@ tokenizeHex input = if isPrefix && len > 0
|
|||||||
len = length numberStr
|
len = length numberStr
|
||||||
numberStr = takeWhile Char.isHexDigit (drop 2 input)
|
numberStr = takeWhile Char.isHexDigit (drop 2 input)
|
||||||
|
|
||||||
|
tokenizeChar :: Tokenizer
|
||||||
|
tokenizeChar ('\'':'\\':x:'\'':_) = seq >>= (\s -> return $ TokenizeResult (IntLiteral s) 4)
|
||||||
|
where
|
||||||
|
seq = case x of
|
||||||
|
'n' -> Just 10
|
||||||
|
't' -> Just 9
|
||||||
|
'v' -> Just 11
|
||||||
|
'b' -> Just 8
|
||||||
|
'r' -> Just 13
|
||||||
|
'f' -> Just 12
|
||||||
|
'a' -> Just 7
|
||||||
|
'\\' -> Just 92
|
||||||
|
'\'' -> Just 39
|
||||||
|
'0' -> Just 0
|
||||||
|
_ -> Nothing
|
||||||
|
tokenizeChar ('\'':x:'\'':_) = Just $ TokenizeResult (IntLiteral . ord $ x) 3
|
||||||
|
tokenizeChar _ = Nothing
|
||||||
|
|
||||||
tokenizeComment :: Tokenizer
|
tokenizeComment :: Tokenizer
|
||||||
tokenizeComment [] = Nothing
|
tokenizeComment [] = Nothing
|
||||||
tokenizeComment (x:xs) = if x == ';'
|
tokenizeComment (x:xs) = if x == ';'
|
||||||
@@ -106,4 +124,5 @@ tokenizers = anyTokenizer
|
|||||||
, sepTokenizer Char.isSpace tokenizeOperators
|
, sepTokenizer Char.isSpace tokenizeOperators
|
||||||
, sepTokenizer Char.isSpace tokenizeHex
|
, sepTokenizer Char.isSpace tokenizeHex
|
||||||
, sepTokenizer Char.isSpace tokenizeDecimal
|
, sepTokenizer Char.isSpace tokenizeDecimal
|
||||||
|
, sepTokenizer Char.isSpace tokenizeChar
|
||||||
]
|
]
|
||||||
Reference in New Issue
Block a user