bartek/smnp-py
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Enable tokenizer to support separators between keywords and integers

Browse Source
This commit is contained in:
Bartłomiej Pluta
2019-07-06 13:35:21 +02:00
parent 675b1774fe
commit 9c4046ac2a
14 changed files with 90 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
smnp/token/tokenizers/bracket.py
View File

@@ -1,8 +1,10 @@
from smnp.token.tools import tokenizeChar
from smnp.token.tools import charTokenizer
from smnp.token.type import TokenType
def tokenizeOpenBracket(input, current, line):
return tokenizeChar(TokenType.OPEN_BRACKET, '{', input, current, line)
return charTokenizer(TokenType.OPEN_BRACKET, '{')(input, current, line)
def tokenizeCloseBracket(input, current, line):
return tokenizeChar(TokenType.CLOSE_BRACKET, '}', input, current, line)
return charTokenizer(TokenType.CLOSE_BRACKET, '}')(input, current, line)