Improve a little bit string tokenizer

2019-07-12 16:52:26 +02:00
parent c7e90b9fbd
commit b711b6a582
1 changed files with 1 additions and 1 deletions
--- a/smnp/token/tokenizers/string.py
+++ b/smnp/token/tokenizers/string.py
@@ -13,5 +13,5 @@ def stringTokenizer(input, current, line):
            char = input[current + consumedChars]
            value += char
            consumedChars += 1
-        return (consumedChars, Token(TokenType.STRING, value[1:len(value)-1], (line, current)))
+        return (consumedChars, Token(TokenType.STRING, value[1:len(value)-1], (line, current), value))
    return (0, None)