Refactor decimal and comment tokenizers

This commit is contained in:
2021-11-03 12:49:26 +01:00
parent 4a5f6b5e64
commit aa984281c0

View File

@@ -54,11 +54,7 @@ tokenizeDecimal input = if null numberStr
where where
number = read numberStr number = read numberStr
len = length numberStr len = length numberStr
numberStr = toNumber input numberStr = takeWhile Char.isDigit input
toNumber [] = []
toNumber (x:xs) = if Char.isDigit x
then x : toNumber xs
else []
tokenizeComment :: Tokenizer tokenizeComment :: Tokenizer
tokenizeComment [] = Nothing tokenizeComment [] = Nothing
@@ -67,11 +63,7 @@ tokenizeComment (x:xs) = if x == ';'
else Nothing else Nothing
where where
len = length comment len = length comment
comment = toEOL xs comment = takeWhile (/='\n') xs
toEOL [] = []
toEOL (x:xs) = if x == '\n'
then []
else x : toEOL xs
type SeparatorPredicate = Char -> Bool type SeparatorPredicate = Char -> Bool
sepTokenizer :: SeparatorPredicate -> Tokenizer -> Tokenizer sepTokenizer :: SeparatorPredicate -> Tokenizer -> Tokenizer